|
@ -244,11 +244,11 @@ class CoNLLParser(FileParser): |
|
|
def parse(self, file): |
|
|
def parse(self, file): |
|
|
words, tags = [], [] |
|
|
words, tags = [], [] |
|
|
data = [] |
|
|
data = [] |
|
|
|
|
|
file = io.TextIOWrapper(file, encoding='utf-8') |
|
|
for i, line in enumerate(file, start=1): |
|
|
for i, line in enumerate(file, start=1): |
|
|
if len(data) >= settings.IMPORT_BATCH_SIZE: |
|
|
if len(data) >= settings.IMPORT_BATCH_SIZE: |
|
|
yield data |
|
|
yield data |
|
|
data = [] |
|
|
data = [] |
|
|
line = line.decode('utf-8') |
|
|
|
|
|
line = line.strip() |
|
|
line = line.strip() |
|
|
if line: |
|
|
if line: |
|
|
try: |
|
|
try: |
|
|