Browse Source

Transparently decode file

pull/223/head
Clemens Wolff 5 years ago
parent
commit
e1ae68c5df
1 changed files with 1 additions and 1 deletions
  1. 2
      app/server/utils.py

2
app/server/utils.py

@ -244,11 +244,11 @@ class CoNLLParser(FileParser):
def parse(self, file): def parse(self, file):
words, tags = [], [] words, tags = [], []
data = [] data = []
file = io.TextIOWrapper(file, encoding='utf-8')
for i, line in enumerate(file, start=1): for i, line in enumerate(file, start=1):
if len(data) >= settings.IMPORT_BATCH_SIZE: if len(data) >= settings.IMPORT_BATCH_SIZE:
yield data yield data
data = [] data = []
line = line.decode('utf-8')
line = line.strip() line = line.strip()
if line: if line:
try: try:

Loading…
Cancel
Save