From 3a2d2a73aca2fa2215b99f2dcd7bf537c810dcd2 Mon Sep 17 00:00:00 2001 From: Clemens Wolff Date: Fri, 25 Jan 2019 09:22:20 -0500 Subject: [PATCH 1/2] Add flake8 linter to CI --- .dockerignore | 1 + .flake8 | 3 +++ requirements.txt | 1 + tools/ci.sh | 1 + 4 files changed, 6 insertions(+) create mode 100644 .flake8 diff --git a/.dockerignore b/.dockerignore index b693ba1b..71043729 100644 --- a/.dockerignore +++ b/.dockerignore @@ -5,4 +5,5 @@ app/db.sqlite3 !data/ !tests/ !tools/ +!.flake8 !requirements.txt diff --git a/.flake8 b/.flake8 new file mode 100644 index 00000000..21c000e8 --- /dev/null +++ b/.flake8 @@ -0,0 +1,3 @@ +[flake8] +filename = app/* +max-line-length = 120 diff --git a/requirements.txt b/requirements.txt index 61264e6e..bbbb681a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,6 +6,7 @@ django-widget-tweaks==1.4.2 djangorestframework==3.8.2 djangorestframework-filters==0.10.2 Faker==0.8.8 +flake8==3.6.0 gunicorn==19.9.0 mixer==6.1.3 psycopg2==2.7.5 diff --git a/tools/ci.sh b/tools/ci.sh index 5bdb9590..8342620e 100755 --- a/tools/ci.sh +++ b/tools/ci.sh @@ -2,6 +2,7 @@ set -o errexit +flake8 python app/manage.py migrate python app/manage.py collectstatic python app/manage.py test server.tests From b5ec59693a9e9d24d908bea8b5853aaa4e711210 Mon Sep 17 00:00:00 2001 From: Clemens Wolff Date: Fri, 25 Jan 2019 09:22:30 -0500 Subject: [PATCH 2/2] Fix all flake8 violations --- app/app/settings.py | 2 +- app/server/tests/test_annotation_api.py | 2 +- app/server/tests/test_doc_api.py | 4 ++-- app/server/tests/test_label_api.py | 4 ++-- app/server/tests/test_models.py | 2 +- app/server/tests/test_projects_api.py | 4 ++-- app/server/tests/test_views.py | 2 +- app/server/views.py | 18 +++++++++--------- 8 files changed, 19 insertions(+), 19 deletions(-) diff --git a/app/app/settings.py b/app/app/settings.py index 247eca68..2c40a6f6 100644 --- a/app/app/settings.py +++ b/app/app/settings.py @@ -159,7 +159,7 @@ DATABASES['default'].update(dj_database_url.config(conn_max_age=500, ssl_require SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_PROTO', 'https') # Allow all host headers -#ALLOWED_HOSTS = ['*'] +# ALLOWED_HOSTS = ['*'] # Size of the batch for creating documents # on the import phase diff --git a/app/server/tests/test_annotation_api.py b/app/server/tests/test_annotation_api.py index d3ae11dd..e51095ad 100644 --- a/app/server/tests/test_annotation_api.py +++ b/app/server/tests/test_annotation_api.py @@ -2,7 +2,7 @@ from django.urls import reverse from rest_framework import status from rest_framework.test import APITestCase from mixer.backend.django import mixer -from ..models import * +from ..models import User, Project class TestAnnotationAPI(APITestCase): diff --git a/app/server/tests/test_doc_api.py b/app/server/tests/test_doc_api.py index d3bfba02..8496e2ad 100644 --- a/app/server/tests/test_doc_api.py +++ b/app/server/tests/test_doc_api.py @@ -2,7 +2,7 @@ from django.urls import reverse from rest_framework import status from rest_framework.test import APITestCase from mixer.backend.django import mixer -from ..models import * +from ..models import User class TestDocAPI(APITestCase): @@ -53,7 +53,7 @@ class TestDocAPI(APITestCase): """ Ensure we cannot get label objects by other. """ - user = self.create_user() + user = self.create_user() # noqa: F841 project = self.create_project() doc = self.create_doc() project.documents.add(doc) diff --git a/app/server/tests/test_label_api.py b/app/server/tests/test_label_api.py index ca28a8b8..04a9c89f 100644 --- a/app/server/tests/test_label_api.py +++ b/app/server/tests/test_label_api.py @@ -2,7 +2,7 @@ from django.urls import reverse from rest_framework import status from rest_framework.test import APITestCase from mixer.backend.django import mixer -from ..models import * +from ..models import User, Label class TestLabelAPI(APITestCase): @@ -56,7 +56,7 @@ class TestLabelAPI(APITestCase): """ Ensure we cannot get label objects by other. """ - user = self.create_user() + user = self.create_user() # noqa: F841 project = self.create_project() label = self.create_label() project.labels.add(label) diff --git a/app/server/tests/test_models.py b/app/server/tests/test_models.py index 3ca4f93d..d81b3348 100644 --- a/app/server/tests/test_models.py +++ b/app/server/tests/test_models.py @@ -2,7 +2,7 @@ from django.test import TestCase from django.core.exceptions import ValidationError from django.db.utils import IntegrityError from mixer.backend.django import mixer -from server.models import Label, DocumentAnnotation, SequenceAnnotation, Seq2seqAnnotation +from ..models import Label, DocumentAnnotation, SequenceAnnotation, Seq2seqAnnotation class TestProject(TestCase): diff --git a/app/server/tests/test_projects_api.py b/app/server/tests/test_projects_api.py index 0ce61653..be9d4081 100644 --- a/app/server/tests/test_projects_api.py +++ b/app/server/tests/test_projects_api.py @@ -2,7 +2,7 @@ from django.urls import reverse from rest_framework import status from rest_framework.test import APITestCase from mixer.backend.django import mixer -from ..models import * +from ..models import User, Project class TestProjects(APITestCase): @@ -14,7 +14,7 @@ class TestProjects(APITestCase): cls.super_username = 'super' cls.normal_user = User.objects.create_user(username=cls.username, password=cls.password) cls.super_user = User.objects.create_superuser(username=cls.super_username, - password=cls.password, email='fizz@buzz.com') + password=cls.password, email='fizz@buzz.com') cls.project1 = mixer.blend('server.Project', project_type=Project.DOCUMENT_CLASSIFICATION, users=[cls.normal_user, cls.super_user]) cls.project2 = mixer.blend('server.Project', project_type=Project.DOCUMENT_CLASSIFICATION, diff --git a/app/server/tests/test_views.py b/app/server/tests/test_views.py index 829445b4..56b333cf 100644 --- a/app/server/tests/test_views.py +++ b/app/server/tests/test_views.py @@ -3,7 +3,7 @@ from django.urls import reverse from django.test import TestCase, Client from rest_framework import status from mixer.backend.django import mixer -from ..models import * +from ..models import User, Document class TestUpload(TestCase): diff --git a/app/server/views.py b/app/server/views.py index bcb1d5ba..4a0c66f3 100644 --- a/app/server/views.py +++ b/app/server/views.py @@ -20,6 +20,7 @@ from app import settings logger = logging.getLogger(__name__) + class IndexView(TemplateView): template_name = 'index.html' @@ -65,13 +66,13 @@ class DataUpload(SuperUserMixin, LoginRequiredMixin, TemplateView): self.message = message def extract_metadata_csv(self, row, text_col, header_without_text): - vals_without_text = [val for i,val in enumerate(row) if i != text_col] + vals_without_text = [val for i, val in enumerate(row) if i != text_col] return json.dumps(dict(zip(header_without_text, vals_without_text))) def csv_to_documents(self, project, file, text_key='text'): form_data = TextIOWrapper(file, encoding='utf-8') reader = csv.reader(form_data) - + maybe_header = next(reader) if maybe_header: if text_key in maybe_header: @@ -82,12 +83,12 @@ class DataUpload(SuperUserMixin, LoginRequiredMixin, TemplateView): else: raise DataUpload.ImportFileError("CSV file must have either a title with \"text\" column or have only one column ") - header_without_text = [title for i,title in enumerate(maybe_header) + header_without_text = [title for i, title in enumerate(maybe_header) if i != text_col] return ( Document( - text=row[text_col], + text=row[text_col], metadata=self.extract_metadata_csv(row, text_col, header_without_text), project=project ) @@ -101,15 +102,14 @@ class DataUpload(SuperUserMixin, LoginRequiredMixin, TemplateView): del copy[text_key] return json.dumps(copy) - def json_to_documents(self, project, file, text_key='text'): + def json_to_documents(self, project, file, text_key='text'): parsed_entries = (json.loads(line) for line in file) - + return ( Document(text=entry[text_key], metadata=self.extract_metadata_json(entry, text_key), project=project) for entry in parsed_entries ) - def post(self, request, *args, **kwargs): project = get_object_or_404(Project, pk=kwargs.get('project_id')) import_format = request.POST['format'] @@ -118,7 +118,7 @@ class DataUpload(SuperUserMixin, LoginRequiredMixin, TemplateView): documents = [] if import_format == 'csv': documents = self.csv_to_documents(project, file) - + elif import_format == 'json': documents = self.json_to_documents(project, file) @@ -175,7 +175,7 @@ class DataDownloadFile(SuperUserMixin, LoginRequiredMixin, View): response['Content-Disposition'] = 'attachment; filename="{}.json"'.format(filename) for d in docs: dump = json.dumps(d.to_json(), ensure_ascii=False) - response.write(dump + '\n') # write each json object end with a newline + response.write(dump + '\n') # write each json object end with a newline return response