Browse Source

fix: use unicode error symbol � instead of failing to import data

pull/2296/head
spodlesny 1 year ago
parent
commit
f092a291d6
1 changed files with 2 additions and 2 deletions
  1. 4
      backend/data_import/pipeline/parsers.py

4
backend/data_import/pipeline/parsers.py

@ -136,8 +136,8 @@ class TextFileParser(Parser):
def parse(self, filename: str) -> Iterator[Dict[Any, Any]]:
encoding = decide_encoding(filename, self.encoding)
with open(filename, encoding=encoding) as f:
yield {DEFAULT_TEXT_COLUMN: f.read()}
with open(filename, encoding=encoding, errors="replace") as f:
yield {DEFAULT_TEXT_COLUMN: f.read().replace("\x00", "\uFFFD")}
class CSVParser(Parser):

Loading…
Cancel
Save