From 7b180e325acd63adbb9bedd033a7f8743b104566 Mon Sep 17 00:00:00 2001 From: Hironsan Date: Fri, 28 Jan 2022 11:30:01 +0900 Subject: [PATCH 1/2] Move code related to example to example app --- backend/api/permissions.py | 9 ---- backend/api/serializers.py | 50 +------------------ backend/api/urls.py | 27 +--------- backend/app/settings.py | 1 + backend/app/urls.py | 1 + backend/examples/__init__.py | 0 backend/examples/admin.py | 0 backend/examples/apps.py | 6 +++ backend/{api => examples}/filters.py | 2 +- backend/examples/migrations/__init__.py | 0 backend/examples/models.py | 0 backend/examples/permissions.py | 10 ++++ backend/examples/serializers.py | 50 +++++++++++++++++++ backend/examples/tests/__init__.py | 0 .../api => examples/tests}/test_comment.py | 3 +- .../api => examples/tests}/test_document.py | 5 +- .../tests}/test_example_state.py | 3 +- .../{api => examples}/tests/test_filters.py | 5 +- .../{api => examples}/tests/test_models.py | 3 +- backend/examples/urls.py | 34 +++++++++++++ backend/examples/views/__init__.py | 0 backend/{api => examples}/views/comment.py | 7 ++- backend/{api => examples}/views/example.py | 7 ++- .../{api => examples}/views/example_state.py | 5 +- 24 files changed, 120 insertions(+), 108 deletions(-) create mode 100644 backend/examples/__init__.py create mode 100644 backend/examples/admin.py create mode 100644 backend/examples/apps.py rename backend/{api => examples}/filters.py (96%) create mode 100644 backend/examples/migrations/__init__.py create mode 100644 backend/examples/models.py create mode 100644 backend/examples/permissions.py create mode 100644 backend/examples/serializers.py create mode 100644 backend/examples/tests/__init__.py rename backend/{api/tests/api => examples/tests}/test_comment.py (98%) rename backend/{api/tests/api => examples/tests}/test_document.py (97%) rename backend/{api/tests/api => examples/tests}/test_example_state.py (96%) rename backend/{api => examples}/tests/test_filters.py (95%) rename backend/{api => examples}/tests/test_models.py (98%) create mode 100644 backend/examples/urls.py create mode 100644 backend/examples/views/__init__.py rename backend/{api => examples}/views/comment.py (91%) rename backend/{api => examples}/views/example.py (93%) rename backend/{api => examples}/views/example_state.py (90%) diff --git a/backend/api/permissions.py b/backend/api/permissions.py index e2d6596b..456c4fe9 100644 --- a/backend/api/permissions.py +++ b/backend/api/permissions.py @@ -1,15 +1,6 @@ from rest_framework.permissions import BasePermission -class IsOwnComment(BasePermission): - @classmethod - def has_object_permission(cls, request, view, obj): - if request.user.is_superuser: - return True - - return obj.user.id == request.user.id - - class IsStaff(BasePermission): def has_permission(self, request, view): if request.user.is_superuser or request.user.is_staff: diff --git a/backend/api/serializers.py b/backend/api/serializers.py index 7bfc890e..7e112b85 100644 --- a/backend/api/serializers.py +++ b/backend/api/serializers.py @@ -1,21 +1,12 @@ from rest_framework import serializers from rest_polymorphic.serializers import PolymorphicSerializer -from .models import (Comment, Example, ExampleState, - ImageClassificationProject, +from .models import (ImageClassificationProject, IntentDetectionAndSlotFillingProject, Project, Seq2seqProject, SequenceLabelingProject, Speech2textProject, Tag, TextClassificationProject) -class CommentSerializer(serializers.ModelSerializer): - - class Meta: - model = Comment - fields = ('id', 'user', 'username', 'example', 'text', 'created_at', ) - read_only_fields = ('user', 'example') - - class TagSerializer(serializers.ModelSerializer): class Meta: @@ -24,45 +15,6 @@ class TagSerializer(serializers.ModelSerializer): read_only_fields = ('id', 'project') -class ExampleSerializer(serializers.ModelSerializer): - annotation_approver = serializers.SerializerMethodField() - is_confirmed = serializers.SerializerMethodField() - - @classmethod - def get_annotation_approver(cls, instance): - approver = instance.annotations_approved_by - return approver.username if approver else None - - def get_is_confirmed(self, instance): - user = self.context.get('request').user - if instance.project.collaborative_annotation: - states = instance.states.all() - else: - states = instance.states.filter(confirmed_by_id=user.id) - return states.count() > 0 - - class Meta: - model = Example - fields = [ - 'id', - 'filename', - 'meta', - 'annotation_approver', - 'comment_count', - 'text', - 'is_confirmed' - ] - read_only_fields = ['filename', 'is_confirmed'] - - -class ExampleStateSerializer(serializers.ModelSerializer): - - class Meta: - model = ExampleState - fields = ('id', 'example', 'confirmed_by') - read_only_fields = ('id', 'example', 'confirmed_by') - - class ProjectSerializer(serializers.ModelSerializer): tags = TagSerializer(many=True, required=False) diff --git a/backend/api/urls.py b/backend/api/urls.py index 3115be8b..7370c4e7 100644 --- a/backend/api/urls.py +++ b/backend/api/urls.py @@ -1,18 +1,8 @@ from django.urls import include, path -from .views import comment, example, example_state, health, project, tag, task +from .views import health, project, tag, task urlpatterns_project = [ - path( - route='examples', - view=example.ExampleList.as_view(), - name='example_list' - ), - path( - route='examples/', - view=example.ExampleDetail.as_view(), - name='example_detail' - ), path( route='tags', view=tag.TagList.as_view(), @@ -23,21 +13,6 @@ urlpatterns_project = [ view=tag.TagDetail.as_view(), name='tag_detail' ), - path( - route='comments', - view=comment.CommentList.as_view(), - name='comment_list' - ), - path( - route='comments/', - view=comment.CommentDetail.as_view(), - name='comment_detail' - ), - path( - route='examples//states', - view=example_state.ExampleStateList.as_view(), - name='example_state_list' - ), ] urlpatterns = [ diff --git a/backend/app/settings.py b/backend/app/settings.py index adf1fb59..981519a0 100644 --- a/backend/app/settings.py +++ b/backend/app/settings.py @@ -61,6 +61,7 @@ INSTALLED_APPS = [ 'auto_labeling.apps.AutoLabelingConfig', 'labels.apps.LabelsConfig', 'label_types.apps.LabelTypesConfig', + 'examples.apps.ExamplesConfig', 'rest_framework', 'rest_framework.authtoken', 'django_filters', diff --git a/backend/app/urls.py b/backend/app/urls.py index 1c7964d1..c5fb3fd3 100644 --- a/backend/app/urls.py +++ b/backend/app/urls.py @@ -48,6 +48,7 @@ urlpatterns += [ path('v1/projects//', include('members.urls')), path('v1/projects//metrics/', include('metrics.urls')), path('v1/projects//', include('auto_labeling.urls')), + path('v1/projects//', include('examples.urls')), path('v1/projects//', include('labels.urls')), path('v1/projects//', include('label_types.urls')), path('swagger/', schema_view.with_ui('swagger', cache_timeout=0), name='schema-swagger-ui'), diff --git a/backend/examples/__init__.py b/backend/examples/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/backend/examples/admin.py b/backend/examples/admin.py new file mode 100644 index 00000000..e69de29b diff --git a/backend/examples/apps.py b/backend/examples/apps.py new file mode 100644 index 00000000..d6f5703a --- /dev/null +++ b/backend/examples/apps.py @@ -0,0 +1,6 @@ +from django.apps import AppConfig + + +class ExamplesConfig(AppConfig): + default_auto_field = 'django.db.models.BigAutoField' + name = 'examples' diff --git a/backend/api/filters.py b/backend/examples/filters.py similarity index 96% rename from backend/api/filters.py rename to backend/examples/filters.py index e035fc65..3fd1954d 100644 --- a/backend/api/filters.py +++ b/backend/examples/filters.py @@ -1,7 +1,7 @@ from django.db.models import Count, Q from django_filters.rest_framework import BooleanFilter, FilterSet -from .models import Example +from api.models import Example class ExampleFilter(FilterSet): diff --git a/backend/examples/migrations/__init__.py b/backend/examples/migrations/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/backend/examples/models.py b/backend/examples/models.py new file mode 100644 index 00000000..e69de29b diff --git a/backend/examples/permissions.py b/backend/examples/permissions.py new file mode 100644 index 00000000..258a1654 --- /dev/null +++ b/backend/examples/permissions.py @@ -0,0 +1,10 @@ +from rest_framework.permissions import BasePermission + + +class IsOwnComment(BasePermission): + @classmethod + def has_object_permission(cls, request, view, obj): + if request.user.is_superuser: + return True + + return obj.user.id == request.user.id diff --git a/backend/examples/serializers.py b/backend/examples/serializers.py new file mode 100644 index 00000000..de009000 --- /dev/null +++ b/backend/examples/serializers.py @@ -0,0 +1,50 @@ +from rest_framework import serializers + +from api.models import Comment, Example, ExampleState + + +class CommentSerializer(serializers.ModelSerializer): + + class Meta: + model = Comment + fields = ('id', 'user', 'username', 'example', 'text', 'created_at', ) + read_only_fields = ('user', 'example') + + +class ExampleSerializer(serializers.ModelSerializer): + annotation_approver = serializers.SerializerMethodField() + is_confirmed = serializers.SerializerMethodField() + + @classmethod + def get_annotation_approver(cls, instance): + approver = instance.annotations_approved_by + return approver.username if approver else None + + def get_is_confirmed(self, instance): + user = self.context.get('request').user + if instance.project.collaborative_annotation: + states = instance.states.all() + else: + states = instance.states.filter(confirmed_by_id=user.id) + return states.count() > 0 + + class Meta: + model = Example + fields = [ + 'id', + 'filename', + 'meta', + 'annotation_approver', + 'comment_count', + 'text', + 'is_confirmed' + ] + read_only_fields = ['filename', 'is_confirmed'] + + +class ExampleStateSerializer(serializers.ModelSerializer): + + class Meta: + model = ExampleState + fields = ('id', 'example', 'confirmed_by') + read_only_fields = ('id', 'example', 'confirmed_by') diff --git a/backend/examples/tests/__init__.py b/backend/examples/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/backend/api/tests/api/test_comment.py b/backend/examples/tests/test_comment.py similarity index 98% rename from backend/api/tests/api/test_comment.py rename to backend/examples/tests/test_comment.py index aca84d3f..b0abc46e 100644 --- a/backend/api/tests/api/test_comment.py +++ b/backend/examples/tests/test_comment.py @@ -1,8 +1,7 @@ from rest_framework import status from rest_framework.reverse import reverse -from .utils import (CRUDMixin, make_comment, make_doc, make_user, - prepare_project) +from api.tests.api.utils import (CRUDMixin, make_comment, make_doc, make_user, prepare_project) class TestCommentListDocAPI(CRUDMixin): diff --git a/backend/api/tests/api/test_document.py b/backend/examples/tests/test_document.py similarity index 97% rename from backend/api/tests/api/test_document.py rename to backend/examples/tests/test_document.py index 86384f48..d0f068f8 100644 --- a/backend/api/tests/api/test_document.py +++ b/backend/examples/tests/test_document.py @@ -4,9 +4,8 @@ from rest_framework import status from rest_framework.reverse import reverse from api.models import DOCUMENT_CLASSIFICATION - -from .utils import (CRUDMixin, assign_user_to_role, make_doc, - make_example_state, make_user, prepare_project) +from api.tests.api.utils import (CRUDMixin, assign_user_to_role, make_doc, + make_example_state, make_user, prepare_project) class TestExampleListAPI(CRUDMixin): diff --git a/backend/api/tests/api/test_example_state.py b/backend/examples/tests/test_example_state.py similarity index 96% rename from backend/api/tests/api/test_example_state.py rename to backend/examples/tests/test_example_state.py index 42b1b1cd..f659969f 100644 --- a/backend/api/tests/api/test_example_state.py +++ b/backend/examples/tests/test_example_state.py @@ -1,8 +1,7 @@ from rest_framework import status from rest_framework.reverse import reverse -from .utils import (CRUDMixin, make_doc, make_example_state, make_user, - prepare_project) +from api.tests.api.utils import (CRUDMixin, make_doc, make_example_state, make_user, prepare_project) class TestExampleStateList(CRUDMixin): diff --git a/backend/api/tests/test_filters.py b/backend/examples/tests/test_filters.py similarity index 95% rename from backend/api/tests/test_filters.py rename to backend/examples/tests/test_filters.py index 30b2d7d4..000cc8c5 100644 --- a/backend/api/tests/test_filters.py +++ b/backend/examples/tests/test_filters.py @@ -2,10 +2,9 @@ from unittest.mock import MagicMock from django.test import TestCase -from api.filters import ExampleFilter from api.models import Example - -from .api.utils import make_doc, make_example_state, prepare_project +from api.tests.api.utils import make_doc, make_example_state, prepare_project +from examples.filters import ExampleFilter class TestFilterMixin(TestCase): diff --git a/backend/api/tests/test_models.py b/backend/examples/tests/test_models.py similarity index 98% rename from backend/api/tests/test_models.py rename to backend/examples/tests/test_models.py index 8d1bf38f..596598fd 100644 --- a/backend/api/tests/test_models.py +++ b/backend/examples/tests/test_models.py @@ -2,8 +2,7 @@ from django.test import TestCase from model_mommy import mommy from api.models import IMAGE_CLASSIFICATION, SEQUENCE_LABELING, ExampleState - -from .api.utils import prepare_project +from api.tests.api.utils import prepare_project class TestExampleState(TestCase): diff --git a/backend/examples/urls.py b/backend/examples/urls.py new file mode 100644 index 00000000..9c22ac3f --- /dev/null +++ b/backend/examples/urls.py @@ -0,0 +1,34 @@ +from django.urls import path + +from .views.example import ExampleList, ExampleDetail +from .views.comment import CommentList, CommentDetail +from .views.example_state import ExampleStateList + + +urlpatterns = [ + path( + route='examples', + view=ExampleList.as_view(), + name='example_list' + ), + path( + route='examples/', + view=ExampleDetail.as_view(), + name='example_detail' + ), + path( + route='comments', + view=CommentList.as_view(), + name='comment_list' + ), + path( + route='comments/', + view=CommentDetail.as_view(), + name='comment_detail' + ), + path( + route='examples//states', + view=ExampleStateList.as_view(), + name='example_state_list' + ), +] diff --git a/backend/examples/views/__init__.py b/backend/examples/views/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/backend/api/views/comment.py b/backend/examples/views/comment.py similarity index 91% rename from backend/api/views/comment.py rename to backend/examples/views/comment.py index 727637e4..c4800b1e 100644 --- a/backend/api/views/comment.py +++ b/backend/examples/views/comment.py @@ -3,11 +3,10 @@ from rest_framework import filters, generics, status from rest_framework.permissions import IsAuthenticated from rest_framework.response import Response +from api.models import Comment from members.permissions import IsInProjectOrAdmin - -from ..models import Comment -from ..permissions import IsOwnComment -from ..serializers import CommentSerializer +from examples.permissions import IsOwnComment +from examples.serializers import CommentSerializer class CommentList(generics.ListCreateAPIView): diff --git a/backend/api/views/example.py b/backend/examples/views/example.py similarity index 93% rename from backend/api/views/example.py rename to backend/examples/views/example.py index 54c4c669..6fd67b7e 100644 --- a/backend/api/views/example.py +++ b/backend/examples/views/example.py @@ -7,12 +7,11 @@ from rest_framework import filters, generics, status from rest_framework.permissions import IsAuthenticated from rest_framework.response import Response +from api.models import Example, Project +from examples.filters import ExampleFilter +from examples.serializers import ExampleSerializer from members.permissions import IsInProjectReadOnlyOrAdmin -from ..filters import ExampleFilter -from ..models import Example, Project -from ..serializers import ExampleSerializer - class ExampleList(generics.ListCreateAPIView): serializer_class = ExampleSerializer diff --git a/backend/api/views/example_state.py b/backend/examples/views/example_state.py similarity index 90% rename from backend/api/views/example_state.py rename to backend/examples/views/example_state.py index 7dd232bb..963572f8 100644 --- a/backend/api/views/example_state.py +++ b/backend/examples/views/example_state.py @@ -2,11 +2,10 @@ from django.shortcuts import get_object_or_404 from rest_framework import generics from rest_framework.permissions import IsAuthenticated +from api.models import Example, ExampleState, Project +from examples.serializers import ExampleStateSerializer from members.permissions import IsInProjectOrAdmin -from ..models import Example, ExampleState, Project -from ..serializers import ExampleStateSerializer - class ExampleStateList(generics.ListCreateAPIView): serializer_class = ExampleStateSerializer From 35846a42d4d982c578e6d2b66cefde75c261538c Mon Sep 17 00:00:00 2001 From: Hironsan Date: Fri, 28 Jan 2022 14:48:37 +0900 Subject: [PATCH 2/2] Move model related to example to example app --- backend/api/admin.py | 18 +--- .../api/migrations/0034_auto_20220128_0246.py | 46 ++++++++++ .../api/migrations/0035_auto_20220128_0246.py | 34 ++++++++ backend/api/models.py | 79 ------------------ backend/auto_labeling/pipeline/labels.py | 3 +- backend/data_export/pipeline/repositories.py | 3 +- backend/data_import/pipeline/data.py | 3 +- backend/data_import/pipeline/writers.py | 3 +- backend/data_import/tests/test_tasks.py | 3 +- backend/examples/admin.py | 19 +++++ backend/examples/filters.py | 2 +- backend/{api => examples}/managers.py | 0 backend/examples/migrations/0001_initial.py | 79 ++++++++++++++++++ backend/examples/models.py | 83 +++++++++++++++++++ backend/examples/serializers.py | 2 +- backend/examples/tests/test_filters.py | 2 +- backend/examples/tests/test_models.py | 3 +- backend/examples/views/comment.py | 2 +- backend/examples/views/example.py | 3 +- backend/examples/views/example_state.py | 3 +- .../migrations/0004_auto_20220128_0246.py | 38 +++++++++ backend/labels/models.py | 3 +- backend/labels/serializers.py | 2 +- backend/metrics/views.py | 2 +- 24 files changed, 325 insertions(+), 110 deletions(-) create mode 100644 backend/api/migrations/0034_auto_20220128_0246.py create mode 100644 backend/api/migrations/0035_auto_20220128_0246.py rename backend/{api => examples}/managers.py (100%) create mode 100644 backend/examples/migrations/0001_initial.py create mode 100644 backend/labels/migrations/0004_auto_20220128_0246.py diff --git a/backend/api/admin.py b/backend/api/admin.py index 941f086b..cee0ae18 100644 --- a/backend/api/admin.py +++ b/backend/api/admin.py @@ -1,13 +1,7 @@ from django.contrib import admin -from .models import (Comment, Example, Project, Seq2seqProject, - SequenceLabelingProject, Tag, TextClassificationProject) - - -class ExampleAdmin(admin.ModelAdmin): - list_display = ('text', 'project', 'meta') - ordering = ('project',) - search_fields = ('text',) +from .models import (Project, Seq2seqProject, SequenceLabelingProject, Tag, + TextClassificationProject) class ProjectAdmin(admin.ModelAdmin): @@ -22,16 +16,8 @@ class TagAdmin(admin.ModelAdmin): search_fields = ('text',) -class CommentAdmin(admin.ModelAdmin): - list_display = ('user', 'example', 'text', 'created_at', ) - ordering = ('user', 'created_at', ) - search_fields = ('user',) - - -admin.site.register(Example, ExampleAdmin) admin.site.register(Project, ProjectAdmin) admin.site.register(TextClassificationProject, ProjectAdmin) admin.site.register(SequenceLabelingProject, ProjectAdmin) admin.site.register(Seq2seqProject, ProjectAdmin) -admin.site.register(Comment, CommentAdmin) admin.site.register(Tag, TagAdmin) diff --git a/backend/api/migrations/0034_auto_20220128_0246.py b/backend/api/migrations/0034_auto_20220128_0246.py new file mode 100644 index 00000000..b6cb5cf1 --- /dev/null +++ b/backend/api/migrations/0034_auto_20220128_0246.py @@ -0,0 +1,46 @@ +# Generated by Django 3.2.11 on 2022-01-28 02:46 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('api', '0033_auto_20220127_0654'), + ] + + operations = [ + migrations.SeparateDatabaseAndState( + state_operations=[ + migrations.RemoveField( + model_name='example', + name='annotations_approved_by', + ), + migrations.RemoveField( + model_name='example', + name='project', + ), + migrations.AlterUniqueTogether( + name='examplestate', + unique_together=None, + ), + migrations.RemoveField( + model_name='examplestate', + name='confirmed_by', + ), + migrations.RemoveField( + model_name='examplestate', + name='example', + ), + migrations.DeleteModel( + name='Comment', + ), + ], + database_operations=[ + migrations.AlterModelTable( + name='Comment', + table='examples_comment' + ) + ] + ) + ] diff --git a/backend/api/migrations/0035_auto_20220128_0246.py b/backend/api/migrations/0035_auto_20220128_0246.py new file mode 100644 index 00000000..700260c5 --- /dev/null +++ b/backend/api/migrations/0035_auto_20220128_0246.py @@ -0,0 +1,34 @@ +# Generated by Django 3.2.11 on 2022-01-28 02:46 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('api', '0034_auto_20220128_0246'), + ('labels', '0004_auto_20220128_0246'), + ] + + operations = [ + migrations.SeparateDatabaseAndState( + state_operations=[ + migrations.DeleteModel( + name='Example', + ), + migrations.DeleteModel( + name='ExampleState', + ), + ], + database_operations=[ + migrations.AlterModelTable( + name='Example', + table='examples_example' + ), + migrations.AlterModelTable( + name='ExampleState', + table='examples_examplestate' + ) + ] + ) + ] diff --git a/backend/api/models.py b/backend/api/models.py index 6dc967a5..760f1133 100644 --- a/backend/api/models.py +++ b/backend/api/models.py @@ -1,12 +1,9 @@ import abc -import uuid from django.contrib.auth.models import User from django.db import models from polymorphic.models import PolymorphicModel -from .managers import ExampleManager, ExampleStateManager - DOCUMENT_CLASSIFICATION = 'DocumentClassification' SEQUENCE_LABELING = 'SequenceLabeling' SEQ2SEQ = 'Seq2seq' @@ -148,82 +145,6 @@ class ImageClassificationProject(Project): return True -class Example(models.Model): - objects = ExampleManager() - - uuid = models.UUIDField(default=uuid.uuid4, editable=False, db_index=True, unique=True) - meta = models.JSONField(default=dict) - filename = models.FileField(default='.', max_length=1024) - project = models.ForeignKey( - to=Project, - on_delete=models.CASCADE, - related_name='examples' - ) - annotations_approved_by = models.ForeignKey( - to=User, - on_delete=models.SET_NULL, - null=True, - blank=True - ) - text = models.TextField(null=True, blank=True) - created_at = models.DateTimeField(auto_now_add=True, db_index=True) - updated_at = models.DateTimeField(auto_now=True) - - @property - def comment_count(self): - return Comment.objects.filter(example=self.id).count() - - @property - def data(self): - if self.project.is_text_project: - return self.text - else: - return str(self.filename) - - class Meta: - ordering = ['created_at'] - - -class ExampleState(models.Model): - objects = ExampleStateManager() - example = models.ForeignKey( - to=Example, - on_delete=models.CASCADE, - related_name='states' - ) - confirmed_by = models.ForeignKey( - to=User, - on_delete=models.CASCADE - ) - confirmed_at = models.DateTimeField(auto_now=True) - - class Meta: - unique_together = (('example', 'confirmed_by'),) - - -class Comment(models.Model): - text = models.TextField() - example = models.ForeignKey( - to=Example, - on_delete=models.CASCADE, - related_name='comments' - ) - user = models.ForeignKey( - to=User, - on_delete=models.CASCADE, - null=True - ) - created_at = models.DateTimeField(auto_now_add=True, db_index=True) - updated_at = models.DateTimeField(auto_now=True) - - @property - def username(self): - return self.user.username - - class Meta: - ordering = ['created_at'] - - class Tag(models.Model): text = models.TextField() project = models.ForeignKey( diff --git a/backend/auto_labeling/pipeline/labels.py b/backend/auto_labeling/pipeline/labels.py index 98c19b9e..7e3c833e 100644 --- a/backend/auto_labeling/pipeline/labels.py +++ b/backend/auto_labeling/pipeline/labels.py @@ -4,7 +4,8 @@ from typing import List from auto_labeling_pipeline.labels import Labels from django.contrib.auth.models import User -from api.models import Project, Example +from api.models import Project +from examples.models import Example from label_types.models import CategoryType, SpanType from labels.models import Label, Category, Span, TextLabel diff --git a/backend/data_export/pipeline/repositories.py b/backend/data_export/pipeline/repositories.py index acb0a069..d0ee0269 100644 --- a/backend/data_export/pipeline/repositories.py +++ b/backend/data_export/pipeline/repositories.py @@ -3,7 +3,8 @@ import itertools from collections import defaultdict from typing import Dict, Iterator, List -from api.models import Example, Project +from api.models import Project +from examples.models import Example from .data import Record diff --git a/backend/data_import/pipeline/data.py b/backend/data_import/pipeline/data.py index 98a36ac7..2bcb7d25 100644 --- a/backend/data_import/pipeline/data.py +++ b/backend/data_import/pipeline/data.py @@ -4,7 +4,8 @@ from typing import Any, Dict from pydantic import BaseModel, validator -from api.models import Example, Project +from api.models import Project +from examples.models import Example class BaseData(BaseModel, abc.ABC): diff --git a/backend/data_import/pipeline/writers.py b/backend/data_import/pipeline/writers.py index e8636b79..e0d3a10c 100644 --- a/backend/data_import/pipeline/writers.py +++ b/backend/data_import/pipeline/writers.py @@ -5,7 +5,8 @@ from typing import Any, Dict, List from django.conf import settings -from api.models import Example, Project +from api.models import Project +from examples.models import Example from label_types.models import CategoryType, SpanType from .exceptions import FileParseException from .readers import BaseReader diff --git a/backend/data_import/tests/test_tasks.py b/backend/data_import/tests/test_tasks.py index fed7da64..a4bafd4c 100644 --- a/backend/data_import/tests/test_tasks.py +++ b/backend/data_import/tests/test_tasks.py @@ -5,7 +5,8 @@ from django.test import TestCase from data_import.celery_tasks import import_dataset from api.models import (DOCUMENT_CLASSIFICATION, INTENT_DETECTION_AND_SLOT_FILLING, SEQ2SEQ, - SEQUENCE_LABELING, Example) + SEQUENCE_LABELING) +from examples.models import Example from label_types.models import CategoryType, SpanType from labels.models import Category, Span from api.tests.api.utils import prepare_project diff --git a/backend/examples/admin.py b/backend/examples/admin.py index e69de29b..e53ffa85 100644 --- a/backend/examples/admin.py +++ b/backend/examples/admin.py @@ -0,0 +1,19 @@ +from django.contrib import admin + +from .models import Example, Comment + + +class ExampleAdmin(admin.ModelAdmin): + list_display = ('text', 'project', 'meta') + ordering = ('project',) + search_fields = ('text',) + + +class CommentAdmin(admin.ModelAdmin): + list_display = ('user', 'example', 'text', 'created_at', ) + ordering = ('user', 'created_at', ) + search_fields = ('user',) + + +admin.site.register(Example, ExampleAdmin) +admin.site.register(Comment, CommentAdmin) diff --git a/backend/examples/filters.py b/backend/examples/filters.py index 3fd1954d..e035fc65 100644 --- a/backend/examples/filters.py +++ b/backend/examples/filters.py @@ -1,7 +1,7 @@ from django.db.models import Count, Q from django_filters.rest_framework import BooleanFilter, FilterSet -from api.models import Example +from .models import Example class ExampleFilter(FilterSet): diff --git a/backend/api/managers.py b/backend/examples/managers.py similarity index 100% rename from backend/api/managers.py rename to backend/examples/managers.py diff --git a/backend/examples/migrations/0001_initial.py b/backend/examples/migrations/0001_initial.py new file mode 100644 index 00000000..3d3556cf --- /dev/null +++ b/backend/examples/migrations/0001_initial.py @@ -0,0 +1,79 @@ +# Generated by Django 3.2.11 on 2022-01-28 02:46 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion +import uuid + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('api', '0034_auto_20220128_0246'), + ] + + operations = [ + migrations.SeparateDatabaseAndState( + state_operations=[ + migrations.CreateModel( + name='Example', + fields=[ + ('id', + models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('uuid', models.UUIDField(db_index=True, default=uuid.uuid4, editable=False, unique=True)), + ('meta', models.JSONField(default=dict)), + ('filename', models.FileField(default='.', max_length=1024, upload_to='')), + ('text', models.TextField(blank=True, null=True)), + ('created_at', models.DateTimeField(auto_now_add=True, db_index=True)), + ('updated_at', models.DateTimeField(auto_now=True)), + ('annotations_approved_by', + models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, + to=settings.AUTH_USER_MODEL)), + ('project', + models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='examples', + to='api.project')), + ], + options={ + 'ordering': ['created_at'], + }, + ), + migrations.CreateModel( + name='Comment', + fields=[ + ('id', + models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('text', models.TextField()), + ('created_at', models.DateTimeField(auto_now_add=True, db_index=True)), + ('updated_at', models.DateTimeField(auto_now=True)), + ('example', + models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='comments', + to='examples.example')), + ('user', models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, + to=settings.AUTH_USER_MODEL)), + ], + options={ + 'ordering': ['created_at'], + }, + ), + migrations.CreateModel( + name='ExampleState', + fields=[ + ('id', + models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('confirmed_at', models.DateTimeField(auto_now=True)), + ('confirmed_by', + models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)), + ('example', + models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='states', + to='examples.example')), + ], + options={ + 'unique_together': {('example', 'confirmed_by')}, + }, + ), + ] + ) + ] diff --git a/backend/examples/models.py b/backend/examples/models.py index e69de29b..6eb3cdc7 100644 --- a/backend/examples/models.py +++ b/backend/examples/models.py @@ -0,0 +1,83 @@ +import uuid + +from django.contrib.auth.models import User +from django.db import models + +from api.models import Project +from .managers import ExampleManager, ExampleStateManager + + +class Example(models.Model): + objects = ExampleManager() + + uuid = models.UUIDField(default=uuid.uuid4, editable=False, db_index=True, unique=True) + meta = models.JSONField(default=dict) + filename = models.FileField(default='.', max_length=1024) + project = models.ForeignKey( + to=Project, + on_delete=models.CASCADE, + related_name='examples' + ) + annotations_approved_by = models.ForeignKey( + to=User, + on_delete=models.SET_NULL, + null=True, + blank=True + ) + text = models.TextField(null=True, blank=True) + created_at = models.DateTimeField(auto_now_add=True, db_index=True) + updated_at = models.DateTimeField(auto_now=True) + + @property + def comment_count(self): + return Comment.objects.filter(example=self.id).count() + + @property + def data(self): + if self.project.is_text_project: + return self.text + else: + return str(self.filename) + + class Meta: + ordering = ['created_at'] + + +class ExampleState(models.Model): + objects = ExampleStateManager() + example = models.ForeignKey( + to=Example, + on_delete=models.CASCADE, + related_name='states' + ) + confirmed_by = models.ForeignKey( + to=User, + on_delete=models.CASCADE + ) + confirmed_at = models.DateTimeField(auto_now=True) + + class Meta: + unique_together = (('example', 'confirmed_by'),) + + +class Comment(models.Model): + text = models.TextField() + example = models.ForeignKey( + to=Example, + on_delete=models.CASCADE, + related_name='comments' + ) + user = models.ForeignKey( + to=User, + on_delete=models.CASCADE, + null=True + ) + created_at = models.DateTimeField(auto_now_add=True, db_index=True) + updated_at = models.DateTimeField(auto_now=True) + + @property + def username(self): + return self.user.username + + class Meta: + ordering = ['created_at'] diff --git a/backend/examples/serializers.py b/backend/examples/serializers.py index de009000..877c6711 100644 --- a/backend/examples/serializers.py +++ b/backend/examples/serializers.py @@ -1,6 +1,6 @@ from rest_framework import serializers -from api.models import Comment, Example, ExampleState +from .models import Example, ExampleState, Comment class CommentSerializer(serializers.ModelSerializer): diff --git a/backend/examples/tests/test_filters.py b/backend/examples/tests/test_filters.py index 000cc8c5..f488a77b 100644 --- a/backend/examples/tests/test_filters.py +++ b/backend/examples/tests/test_filters.py @@ -2,8 +2,8 @@ from unittest.mock import MagicMock from django.test import TestCase -from api.models import Example from api.tests.api.utils import make_doc, make_example_state, prepare_project +from examples.models import Example from examples.filters import ExampleFilter diff --git a/backend/examples/tests/test_models.py b/backend/examples/tests/test_models.py index 596598fd..063d83c2 100644 --- a/backend/examples/tests/test_models.py +++ b/backend/examples/tests/test_models.py @@ -1,8 +1,9 @@ from django.test import TestCase from model_mommy import mommy -from api.models import IMAGE_CLASSIFICATION, SEQUENCE_LABELING, ExampleState +from api.models import IMAGE_CLASSIFICATION, SEQUENCE_LABELING from api.tests.api.utils import prepare_project +from examples.models import ExampleState class TestExampleState(TestCase): diff --git a/backend/examples/views/comment.py b/backend/examples/views/comment.py index c4800b1e..96ed6eae 100644 --- a/backend/examples/views/comment.py +++ b/backend/examples/views/comment.py @@ -3,8 +3,8 @@ from rest_framework import filters, generics, status from rest_framework.permissions import IsAuthenticated from rest_framework.response import Response -from api.models import Comment from members.permissions import IsInProjectOrAdmin +from examples.models import Comment from examples.permissions import IsOwnComment from examples.serializers import CommentSerializer diff --git a/backend/examples/views/example.py b/backend/examples/views/example.py index 6fd67b7e..b7128ea4 100644 --- a/backend/examples/views/example.py +++ b/backend/examples/views/example.py @@ -7,8 +7,9 @@ from rest_framework import filters, generics, status from rest_framework.permissions import IsAuthenticated from rest_framework.response import Response -from api.models import Example, Project +from api.models import Project from examples.filters import ExampleFilter +from examples.models import Example from examples.serializers import ExampleSerializer from members.permissions import IsInProjectReadOnlyOrAdmin diff --git a/backend/examples/views/example_state.py b/backend/examples/views/example_state.py index 963572f8..ea182b43 100644 --- a/backend/examples/views/example_state.py +++ b/backend/examples/views/example_state.py @@ -2,7 +2,8 @@ from django.shortcuts import get_object_or_404 from rest_framework import generics from rest_framework.permissions import IsAuthenticated -from api.models import Example, ExampleState, Project +from api.models import Project +from examples.models import Example, ExampleState from examples.serializers import ExampleStateSerializer from members.permissions import IsInProjectOrAdmin diff --git a/backend/labels/migrations/0004_auto_20220128_0246.py b/backend/labels/migrations/0004_auto_20220128_0246.py new file mode 100644 index 00000000..6c3669be --- /dev/null +++ b/backend/labels/migrations/0004_auto_20220128_0246.py @@ -0,0 +1,38 @@ +# Generated by Django 3.2.11 on 2022-01-28 02:46 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('examples', '0001_initial'), + ('labels', '0003_auto_20220127_0654'), + ] + + operations = [ + migrations.SeparateDatabaseAndState( + state_operations=[ + migrations.AlterField( + model_name='category', + name='example', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='categories', + to='examples.example'), + ), + migrations.AlterField( + model_name='span', + name='example', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='spans', + to='examples.example'), + ), + migrations.AlterField( + model_name='textlabel', + name='example', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='texts', + to='examples.example'), + ), + ], + database_operations=[] + ) + ] diff --git a/backend/labels/models.py b/backend/labels/models.py index 5b8690b5..462bf1ca 100644 --- a/backend/labels/models.py +++ b/backend/labels/models.py @@ -3,7 +3,8 @@ from django.core.exceptions import ValidationError from django.db import models from .managers import LabelManager, CategoryManager, SpanManager, TextLabelManager -from api.models import Example, Project +from api.models import Project +from examples.models import Example from label_types.models import CategoryType, SpanType, RelationType diff --git a/backend/labels/serializers.py b/backend/labels/serializers.py index 0652d267..5d2c145a 100644 --- a/backend/labels/serializers.py +++ b/backend/labels/serializers.py @@ -1,6 +1,6 @@ from rest_framework import serializers -from api.models import Example +from examples.models import Example from label_types.models import CategoryType, SpanType from .models import Category, Span, TextLabel, Relation diff --git a/backend/metrics/views.py b/backend/metrics/views.py index ca2887f6..6effdf08 100644 --- a/backend/metrics/views.py +++ b/backend/metrics/views.py @@ -5,7 +5,7 @@ from rest_framework.permissions import IsAuthenticated from rest_framework.response import Response from rest_framework.views import APIView -from api.models import Example, ExampleState +from examples.models import Example, ExampleState from label_types.models import LabelType, CategoryType, SpanType from labels.models import Label, Category, Span from members.models import Member