Browse Source

Fix TextLineDataset to raise exception

pull/1573/head
Hironsan 3 years ago
parent
commit
8a5b09f053
3 changed files with 13 additions and 5 deletions
  1. 1
      backend/api/tests/data/example.txt
  2. 2
      backend/api/tests/test_tasks.py
  3. 15
      backend/api/views/upload/dataset.py

1
backend/api/tests/data/example.txt

@ -1,3 +1,4 @@
exampleA
exampleB
exampleC

2
backend/api/tests/test_tasks.py

@ -107,7 +107,7 @@ class TestIngestClassificationData(TestIngestData):
filename = 'example.txt'
file_format = 'TextFile'
dataset = [
('exampleA\nexampleB\nexampleC\n', [])
('exampleA\nexampleB\n\nexampleC\n', [])
]
self.ingest_data(filename, file_format)
self.assert_examples(dataset)

15
backend/api/views/upload/dataset.py

@ -153,11 +153,18 @@ class TextLineDataset(Dataset):
def load(self, filename: str) -> Iterator[Record]:
encoding = self.detect_encoding(filename)
errors = []
with open(filename, encoding=encoding) as f:
for line in f:
data = self.data_class.parse(filename=filename, text=line.rstrip())
record = Record(data=data)
yield record
for line_num, line in enumerate(f, start=1):
try:
data = self.data_class.parse(filename=filename, text=line.rstrip())
record = Record(data=data)
yield record
except ValidationError:
message = 'The empty text is not allowed.'
errors.append(FileParseException(filename, line_num, message))
if errors:
raise FileParseExceptions(errors)
class CsvDataset(Dataset):

Loading…
Cancel
Save