|
@ -64,7 +64,7 @@ class CsvWriter(BaseWriter): |
|
|
for record in records: |
|
|
for record in records: |
|
|
filename = os.path.join(self.tmpdir, f'{record.user}.{self.extension}') |
|
|
filename = os.path.join(self.tmpdir, f'{record.user}.{self.extension}') |
|
|
if filename not in writers: |
|
|
if filename not in writers: |
|
|
f = open(filename, mode='a') |
|
|
|
|
|
|
|
|
f = open(filename, mode='a', encoding='utf-8') |
|
|
writer = csv.DictWriter(f, header) |
|
|
writer = csv.DictWriter(f, header) |
|
|
writer.writeheader() |
|
|
writer.writeheader() |
|
|
writers[filename] = writer |
|
|
writers[filename] = writer |
|
@ -72,11 +72,12 @@ class CsvWriter(BaseWriter): |
|
|
writer = writers[filename] |
|
|
writer = writers[filename] |
|
|
line = self.create_line(record) |
|
|
line = self.create_line(record) |
|
|
writer.writerow(line) |
|
|
writer.writerow(line) |
|
|
|
|
|
|
|
|
|
|
|
for f in file_handlers: |
|
|
|
|
|
f.close() |
|
|
save_file = self.write_zip(writers) |
|
|
save_file = self.write_zip(writers) |
|
|
for file in writers: |
|
|
for file in writers: |
|
|
os.remove(file) |
|
|
os.remove(file) |
|
|
for f in file_handlers: |
|
|
|
|
|
f.close() |
|
|
|
|
|
return save_file |
|
|
return save_file |
|
|
|
|
|
|
|
|
def create_line(self, record) -> Dict: |
|
|
def create_line(self, record) -> Dict: |
|
@ -102,14 +103,14 @@ class JSONWriter(BaseWriter): |
|
|
for record in records: |
|
|
for record in records: |
|
|
filename = os.path.join(self.tmpdir, f'{record.user}.{self.extension}') |
|
|
filename = os.path.join(self.tmpdir, f'{record.user}.{self.extension}') |
|
|
if filename not in writers: |
|
|
if filename not in writers: |
|
|
f = open(filename, mode='a') |
|
|
|
|
|
|
|
|
f = open(filename, mode='a', encoding='utf-8') |
|
|
writers[filename] = f |
|
|
writers[filename] = f |
|
|
line = self.create_line(record) |
|
|
line = self.create_line(record) |
|
|
contents[filename].append(line) |
|
|
contents[filename].append(line) |
|
|
|
|
|
|
|
|
for filename, f in writers.items(): |
|
|
for filename, f in writers.items(): |
|
|
content = contents[filename] |
|
|
content = contents[filename] |
|
|
json.dump(content, f) |
|
|
|
|
|
|
|
|
json.dump(content, f, ensure_ascii=False) |
|
|
f.close() |
|
|
f.close() |
|
|
|
|
|
|
|
|
save_file = self.write_zip(writers) |
|
|
save_file = self.write_zip(writers) |
|
@ -135,7 +136,7 @@ class JSONLWriter(LineWriter): |
|
|
'data': record.data, |
|
|
'data': record.data, |
|
|
'label': record.label, |
|
|
'label': record.label, |
|
|
**record.metadata |
|
|
**record.metadata |
|
|
}) |
|
|
|
|
|
|
|
|
}, ensure_ascii=False) |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class FastTextWriter(LineWriter): |
|
|
class FastTextWriter(LineWriter): |
|
|