From 7a3f9b616b9f3946be91d55a52497505febad75c Mon Sep 17 00:00:00 2001 From: Hironsan Date: Thu, 20 Jan 2022 13:15:57 +0900 Subject: [PATCH 1/2] Move auto labeling code to auto labeling app --- Pipfile | 2 +- backend/api/admin.py | 18 ++----- backend/api/exceptions.py | 30 +---------- backend/api/serializers.py | 53 +----------------- backend/api/tests/api/test_annotation.py | 5 +- backend/api/tests/api/test_document.py | 3 +- backend/api/tests/api/test_label.py | 3 +- backend/api/tests/api/utils.py | 7 ++- backend/api/tests/test_filters.py | 5 +- backend/api/tests/test_models.py | 6 ++- backend/api/urls.py | 49 +---------------- backend/app/settings.py | 1 + backend/app/urls.py | 1 + backend/auto_labeling/__init__.py | 0 backend/auto_labeling/admin.py | 17 ++++++ backend/auto_labeling/apps.py | 6 +++ backend/auto_labeling/exceptions.py | 29 ++++++++++ backend/auto_labeling/migrations/__init__.py | 0 backend/auto_labeling/models.py | 0 backend/auto_labeling/serializers.py | 54 +++++++++++++++++++ backend/auto_labeling/tests/__init__.py | 0 .../tests/test_views.py} | 16 +++--- backend/auto_labeling/urls.py | 53 ++++++++++++++++++ .../views.py} | 12 ++--- backend/members/tests.py | 2 +- 25 files changed, 200 insertions(+), 172 deletions(-) create mode 100644 backend/auto_labeling/__init__.py create mode 100644 backend/auto_labeling/admin.py create mode 100644 backend/auto_labeling/apps.py create mode 100644 backend/auto_labeling/exceptions.py create mode 100644 backend/auto_labeling/migrations/__init__.py create mode 100644 backend/auto_labeling/models.py create mode 100644 backend/auto_labeling/serializers.py create mode 100644 backend/auto_labeling/tests/__init__.py rename backend/{api/tests/api/test_auto_labeling.py => auto_labeling/tests/test_views.py} (88%) create mode 100644 backend/auto_labeling/urls.py rename backend/{api/views/auto_labeling.py => auto_labeling/views.py} (96%) diff --git a/Pipfile b/Pipfile index cf93868c..1ffe7096 100644 --- a/Pipfile +++ b/Pipfile @@ -60,6 +60,6 @@ python_version = "3.8" isort = "isort api -c --skip migrations" flake8 = "flake8 --filename \"*.py\" --extend-exclude \"api/migrations\"" wait_for_db = "python manage.py wait_for_db" -test = "python manage.py test api.tests roles.tests members.tests metrics.tests users.tests data_import.tests data_export.tests" +test = "python manage.py test --pattern=\"test*.py\"" migrate = "python manage.py migrate" collectstatic = "python manage.py collectstatic --noinput" diff --git a/backend/api/admin.py b/backend/api/admin.py index 7bac9d23..3c10304d 100644 --- a/backend/api/admin.py +++ b/backend/api/admin.py @@ -1,8 +1,8 @@ from django.contrib import admin -from .models import (AutoLabelingConfig, Category, CategoryType, Comment, - Example, Project, Seq2seqProject, SequenceLabelingProject, - Span, SpanType, Tag, TextClassificationProject, TextLabel) +from .models import (Category, CategoryType, Comment, Example, Project, + Seq2seqProject, SequenceLabelingProject, Span, SpanType, + Tag, TextClassificationProject, TextLabel) class LabelAdmin(admin.ModelAdmin): @@ -58,18 +58,6 @@ class CommentAdmin(admin.ModelAdmin): search_fields = ('user',) -class AutoLabelingConfigAdmin(admin.ModelAdmin): - list_display = ('project', 'model_name', 'model_attrs',) - ordering = ('project',) - - def get_readonly_fields(self, request, obj=None): - if obj: - return ["model_name"] - else: - return [] - - -admin.site.register(AutoLabelingConfig, AutoLabelingConfigAdmin) admin.site.register(Category, CategoryAdmin) admin.site.register(Span, SpanAdmin) admin.site.register(TextLabel, TextLabelAdmin) diff --git a/backend/api/exceptions.py b/backend/api/exceptions.py index bb5b0e94..5c89ebb0 100644 --- a/backend/api/exceptions.py +++ b/backend/api/exceptions.py @@ -1,6 +1,5 @@ from rest_framework import status -from rest_framework.exceptions import (APIException, PermissionDenied, - ValidationError) +from rest_framework.exceptions import APIException class FileParseException(APIException): @@ -13,33 +12,6 @@ class FileParseException(APIException): super().__init__(detail, code) -class AutoLabelingException(APIException): - status_code = status.HTTP_400_BAD_REQUEST - default_detail = 'Auto labeling not allowed for the document with labels.' - - -class AutoLabelingPermissionDenied(PermissionDenied): - default_detail = 'You do not have permission to perform auto labeling.' \ - 'Please ask the project administrators to add you.' - - -class URLConnectionError(ValidationError): - default_detail = 'Failed to establish a connection. Please check the URL or network.' - - -class AWSTokenError(ValidationError): - default_detail = 'The security token included in the request is invalid.' - - -class SampleDataException(ValidationError): - default_detail = 'The response is empty. Maybe the sample data is not appropriate.' \ - 'Please specify another sample data which returns at least one label.' - - -class TemplateMappingError(ValidationError): - default_detail = 'The response cannot be mapped. You might need to change the template.' - - class LabelValidationError(APIException): status_code = status.HTTP_400_BAD_REQUEST default_detail = 'You cannot create a label with same name or shortcut key.' diff --git a/backend/api/serializers.py b/backend/api/serializers.py index c86333a3..f99893c4 100644 --- a/backend/api/serializers.py +++ b/backend/api/serializers.py @@ -1,11 +1,8 @@ -from auto_labeling_pipeline.models import RequestModelFactory from rest_framework import serializers from rest_framework.exceptions import ValidationError from rest_polymorphic.serializers import PolymorphicSerializer -from .models import (DOCUMENT_CLASSIFICATION, IMAGE_CLASSIFICATION, SEQ2SEQ, - SEQUENCE_LABELING, SPEECH2TEXT, AnnotationRelations, - AutoLabelingConfig, Category, CategoryType, Comment, +from .models import (AnnotationRelations, Category, CategoryType, Comment, Example, ExampleState, ImageClassificationProject, IntentDetectionAndSlotFillingProject, Label, Project, RelationTypes, Seq2seqProject, SequenceLabelingProject, @@ -284,54 +281,6 @@ class TextLabelSerializer(serializers.ModelSerializer): read_only_fields = ('user',) -class AutoLabelingConfigSerializer(serializers.ModelSerializer): - - class Meta: - model = AutoLabelingConfig - fields = ('id', 'model_name', 'model_attrs', 'template', 'label_mapping', 'default') - read_only_fields = ('created_at', 'updated_at') - - def validate_model_name(self, value): - try: - RequestModelFactory.find(value) - except NameError: - raise serializers.ValidationError(f'The specified model name {value} does not exist.') - return value - - def valid_label_mapping(self, value): - if isinstance(value, dict): - return value - else: - raise serializers.ValidationError(f'The {value} is not a dictionary. Please specify it as a dictionary.') - - def validate(self, data): - try: - RequestModelFactory.create(data['model_name'], data['model_attrs']) - except Exception: - model = RequestModelFactory.find(data['model_name']) - schema = model.schema() - required_fields = ', '.join(schema['required']) if 'required' in schema else '' - raise serializers.ValidationError( - 'The attributes does not match the model.' - 'You need to correctly specify the required fields: {}'.format(required_fields) - ) - return data - - -def get_annotation_serializer(task: str): - mapping = { - DOCUMENT_CLASSIFICATION: CategorySerializer, - SEQUENCE_LABELING: SpanSerializer, - SEQ2SEQ: TextLabelSerializer, - SPEECH2TEXT: TextLabelSerializer, - IMAGE_CLASSIFICATION: CategorySerializer, - } - try: - return mapping[task] - except KeyError: - raise ValueError(f'{task} is not implemented.') - - class RelationTypesSerializer(serializers.ModelSerializer): def validate(self, attrs): diff --git a/backend/api/tests/api/test_annotation.py b/backend/api/tests/api/test_annotation.py index 17367086..fc20d5ab 100644 --- a/backend/api/tests/api/test_annotation.py +++ b/backend/api/tests/api/test_annotation.py @@ -1,8 +1,9 @@ from rest_framework import status from rest_framework.reverse import reverse -from ...models import (DOCUMENT_CLASSIFICATION, SEQ2SEQ, SEQUENCE_LABELING, - Category, Span, TextLabel) +from api.models import (DOCUMENT_CLASSIFICATION, SEQ2SEQ, SEQUENCE_LABELING, + Category, Span, TextLabel) + from .utils import (CRUDMixin, make_annotation, make_doc, make_label, make_user, prepare_project) diff --git a/backend/api/tests/api/test_document.py b/backend/api/tests/api/test_document.py index 19d9b2cb..e71d6f35 100644 --- a/backend/api/tests/api/test_document.py +++ b/backend/api/tests/api/test_document.py @@ -3,7 +3,8 @@ from django.utils.http import urlencode from rest_framework import status from rest_framework.reverse import reverse -from ...models import DOCUMENT_CLASSIFICATION +from api.models import DOCUMENT_CLASSIFICATION + from .utils import (CRUDMixin, assign_user_to_role, make_doc, make_example_state, make_user, prepare_project) diff --git a/backend/api/tests/api/test_label.py b/backend/api/tests/api/test_label.py index 6e67e3b9..4da5a3e1 100644 --- a/backend/api/tests/api/test_label.py +++ b/backend/api/tests/api/test_label.py @@ -5,7 +5,8 @@ from rest_framework import status from rest_framework.reverse import reverse from rest_framework.test import APITestCase -from ...models import DOCUMENT_CLASSIFICATION +from api.models import DOCUMENT_CLASSIFICATION + from .utils import (DATA_DIR, CRUDMixin, make_label, make_project, make_user, prepare_project) diff --git a/backend/api/tests/api/utils.py b/backend/api/tests/api/utils.py index 4cf9e59f..6c6b3f19 100644 --- a/backend/api/tests/api/utils.py +++ b/backend/api/tests/api/utils.py @@ -8,13 +8,12 @@ from model_mommy import mommy from rest_framework import status from rest_framework.test import APITestCase +from api.models import (DOCUMENT_CLASSIFICATION, IMAGE_CLASSIFICATION, + INTENT_DETECTION_AND_SLOT_FILLING, SEQ2SEQ, + SEQUENCE_LABELING, SPEECH2TEXT) from members.models import Member from roles.models import Role -from ...models import (DOCUMENT_CLASSIFICATION, IMAGE_CLASSIFICATION, - INTENT_DETECTION_AND_SLOT_FILLING, SEQ2SEQ, - SEQUENCE_LABELING, SPEECH2TEXT) - DATA_DIR = os.path.join(os.path.dirname(__file__), '../../../data_import/tests/data') diff --git a/backend/api/tests/test_filters.py b/backend/api/tests/test_filters.py index 4c8d6312..30b2d7d4 100644 --- a/backend/api/tests/test_filters.py +++ b/backend/api/tests/test_filters.py @@ -2,8 +2,9 @@ from unittest.mock import MagicMock from django.test import TestCase -from ..filters import ExampleFilter -from ..models import Example +from api.filters import ExampleFilter +from api.models import Example + from .api.utils import make_doc, make_example_state, prepare_project diff --git a/backend/api/tests/test_models.py b/backend/api/tests/test_models.py index 7ad54a22..8cda0bc8 100644 --- a/backend/api/tests/test_models.py +++ b/backend/api/tests/test_models.py @@ -3,8 +3,10 @@ from django.db.utils import IntegrityError from django.test import TestCase from model_mommy import mommy -from ..models import (SEQUENCE_LABELING, Category, CategoryType, ExampleState, - Span, SpanType, TextLabel, generate_random_hex_color) +from api.models import (SEQUENCE_LABELING, Category, CategoryType, + ExampleState, Span, SpanType, TextLabel, + generate_random_hex_color) + from .api.utils import prepare_project diff --git a/backend/api/urls.py b/backend/api/urls.py index 6068adea..1057388c 100644 --- a/backend/api/urls.py +++ b/backend/api/urls.py @@ -1,7 +1,7 @@ from django.urls import include, path -from .views import (annotation, auto_labeling, comment, example, example_state, - health, label, project, tag, task) +from .views import (annotation, comment, example, example_state, health, label, + project, tag, task) from .views.tasks import category, relation, span, text urlpatterns_project = [ @@ -140,51 +140,6 @@ urlpatterns_project = [ view=example_state.ExampleStateList.as_view(), name='example_state_list' ), - path( - route='auto-labeling-templates', - view=auto_labeling.AutoLabelingTemplateListAPI.as_view(), - name='auto_labeling_templates' - ), - path( - route='auto-labeling-templates/', - view=auto_labeling.AutoLabelingTemplateDetailAPI.as_view(), - name='auto_labeling_template' - ), - path( - route='auto-labeling-configs', - view=auto_labeling.AutoLabelingConfigList.as_view(), - name='auto_labeling_configs' - ), - path( - route='auto-labeling-configs/', - view=auto_labeling.AutoLabelingConfigDetail.as_view(), - name='auto_labeling_config' - ), - path( - route='auto-labeling-config-testing', - view=auto_labeling.AutoLabelingConfigTest.as_view(), - name='auto_labeling_config_test' - ), - path( - route='examples//auto-labeling', - view=auto_labeling.AutoLabelingAnnotation.as_view(), - name='auto_labeling_annotation' - ), - path( - route='auto-labeling-parameter-testing', - view=auto_labeling.AutoLabelingConfigParameterTest.as_view(), - name='auto_labeling_parameter_testing' - ), - path( - route='auto-labeling-template-testing', - view=auto_labeling.AutoLabelingTemplateTest.as_view(), - name='auto_labeling_template_test' - ), - path( - route='auto-labeling-mapping-testing', - view=auto_labeling.AutoLabelingMappingTest.as_view(), - name='auto_labeling_mapping_test' - ) ] urlpatterns = [ diff --git a/backend/app/settings.py b/backend/app/settings.py index 6df44619..8ce096ea 100644 --- a/backend/app/settings.py +++ b/backend/app/settings.py @@ -58,6 +58,7 @@ INSTALLED_APPS = [ 'users.apps.UsersConfig', 'data_import.apps.DataImportConfig', 'data_export.apps.DataExportConfig', + 'auto_labeling.apps.AutoLabelingConfig', 'rest_framework', 'rest_framework.authtoken', 'django_filters', diff --git a/backend/app/urls.py b/backend/app/urls.py index 01ebc8b4..6148d59c 100644 --- a/backend/app/urls.py +++ b/backend/app/urls.py @@ -47,6 +47,7 @@ urlpatterns += [ path('v1/', include('data_export.urls')), path('v1/projects//', include('members.urls')), path('v1/projects//metrics/', include('metrics.urls')), + path('v1/projects//', include('auto_labeling.urls')), path('swagger/', schema_view.with_ui('swagger', cache_timeout=0), name='schema-swagger-ui'), re_path('', TemplateView.as_view(template_name='index.html')), ] diff --git a/backend/auto_labeling/__init__.py b/backend/auto_labeling/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/backend/auto_labeling/admin.py b/backend/auto_labeling/admin.py new file mode 100644 index 00000000..411e850f --- /dev/null +++ b/backend/auto_labeling/admin.py @@ -0,0 +1,17 @@ +from django.contrib import admin + +from api.models import AutoLabelingConfig + + +class AutoLabelingConfigAdmin(admin.ModelAdmin): + list_display = ('project', 'model_name', 'model_attrs',) + ordering = ('project',) + + def get_readonly_fields(self, request, obj=None): + if obj: + return ["model_name"] + else: + return [] + + +admin.site.register(AutoLabelingConfig, AutoLabelingConfigAdmin) diff --git a/backend/auto_labeling/apps.py b/backend/auto_labeling/apps.py new file mode 100644 index 00000000..8368f1b1 --- /dev/null +++ b/backend/auto_labeling/apps.py @@ -0,0 +1,6 @@ +from django.apps import AppConfig + + +class AutoLabelingConfig(AppConfig): + default_auto_field = 'django.db.models.BigAutoField' + name = 'auto_labeling' diff --git a/backend/auto_labeling/exceptions.py b/backend/auto_labeling/exceptions.py new file mode 100644 index 00000000..70c8c1fd --- /dev/null +++ b/backend/auto_labeling/exceptions.py @@ -0,0 +1,29 @@ +from rest_framework import status +from rest_framework.exceptions import APIException, PermissionDenied, ValidationError + + +class AutoLabelingException(APIException): + status_code = status.HTTP_400_BAD_REQUEST + default_detail = 'Auto labeling not allowed for the document with labels.' + + +class AutoLabelingPermissionDenied(PermissionDenied): + default_detail = 'You do not have permission to perform auto labeling.' \ + 'Please ask the project administrators to add you.' + + +class URLConnectionError(ValidationError): + default_detail = 'Failed to establish a connection. Please check the URL or network.' + + +class AWSTokenError(ValidationError): + default_detail = 'The security token included in the request is invalid.' + + +class SampleDataException(ValidationError): + default_detail = 'The response is empty. Maybe the sample data is not appropriate.' \ + 'Please specify another sample data which returns at least one label.' + + +class TemplateMappingError(ValidationError): + default_detail = 'The response cannot be mapped. You might need to change the template.' diff --git a/backend/auto_labeling/migrations/__init__.py b/backend/auto_labeling/migrations/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/backend/auto_labeling/models.py b/backend/auto_labeling/models.py new file mode 100644 index 00000000..e69de29b diff --git a/backend/auto_labeling/serializers.py b/backend/auto_labeling/serializers.py new file mode 100644 index 00000000..857d7d10 --- /dev/null +++ b/backend/auto_labeling/serializers.py @@ -0,0 +1,54 @@ +from auto_labeling_pipeline.models import RequestModelFactory +from rest_framework import serializers + +from api.models import AutoLabelingConfig, DOCUMENT_CLASSIFICATION, SEQUENCE_LABELING, SEQ2SEQ, SPEECH2TEXT, \ + IMAGE_CLASSIFICATION +from api.serializers import CategorySerializer, SpanSerializer, TextLabelSerializer + + +class AutoLabelingConfigSerializer(serializers.ModelSerializer): + + class Meta: + model = AutoLabelingConfig + fields = ('id', 'model_name', 'model_attrs', 'template', 'label_mapping', 'default') + read_only_fields = ('created_at', 'updated_at') + + def validate_model_name(self, value): + try: + RequestModelFactory.find(value) + except NameError: + raise serializers.ValidationError(f'The specified model name {value} does not exist.') + return value + + def valid_label_mapping(self, value): + if isinstance(value, dict): + return value + else: + raise serializers.ValidationError(f'The {value} is not a dictionary. Please specify it as a dictionary.') + + def validate(self, data): + try: + RequestModelFactory.create(data['model_name'], data['model_attrs']) + except Exception: + model = RequestModelFactory.find(data['model_name']) + schema = model.schema() + required_fields = ', '.join(schema['required']) if 'required' in schema else '' + raise serializers.ValidationError( + 'The attributes does not match the model.' + 'You need to correctly specify the required fields: {}'.format(required_fields) + ) + return data + + +def get_annotation_serializer(task: str): + mapping = { + DOCUMENT_CLASSIFICATION: CategorySerializer, + SEQUENCE_LABELING: SpanSerializer, + SEQ2SEQ: TextLabelSerializer, + SPEECH2TEXT: TextLabelSerializer, + IMAGE_CLASSIFICATION: CategorySerializer, + } + try: + return mapping[task] + except KeyError: + raise ValueError(f'{task} is not implemented.') diff --git a/backend/auto_labeling/tests/__init__.py b/backend/auto_labeling/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/backend/api/tests/api/test_auto_labeling.py b/backend/auto_labeling/tests/test_views.py similarity index 88% rename from backend/api/tests/api/test_auto_labeling.py rename to backend/auto_labeling/tests/test_views.py index 2ef44679..8d6eefd4 100644 --- a/backend/api/tests/api/test_auto_labeling.py +++ b/backend/auto_labeling/tests/test_views.py @@ -6,9 +6,9 @@ from auto_labeling_pipeline.models import RequestModelFactory from rest_framework import status from rest_framework.reverse import reverse -from ...models import DOCUMENT_CLASSIFICATION, IMAGE_CLASSIFICATION -from .utils import (CRUDMixin, make_auto_labeling_config, make_doc, make_image, - prepare_project) +from api.models import DOCUMENT_CLASSIFICATION, IMAGE_CLASSIFICATION +from api.tests.api.utils import (CRUDMixin, make_auto_labeling_config, make_doc, make_image, + prepare_project) data_dir = pathlib.Path(__file__).parent / 'data' @@ -24,20 +24,20 @@ class TestConfigParameter(CRUDMixin): } self.url = reverse(viewname='auto_labeling_parameter_testing', args=[self.project.item.id]) - @patch('api.views.auto_labeling.AutoLabelingConfigParameterTest.send_request', return_value={}) + @patch('auto_labeling.views.AutoLabelingConfigParameterTest.send_request', return_value={}) def test_called_with_proper_model(self, mock): self.assert_create(self.project.users[0], status.HTTP_200_OK) _, kwargs = mock.call_args expected = RequestModelFactory.create(self.data['model_name'], self.data['model_attrs']) self.assertEqual(kwargs['model'], expected) - @patch('api.views.auto_labeling.AutoLabelingConfigParameterTest.send_request', return_value={}) + @patch('auto_labeling.views.AutoLabelingConfigParameterTest.send_request', return_value={}) def test_called_with_text(self, mock): self.assert_create(self.project.users[0], status.HTTP_200_OK) _, kwargs = mock.call_args self.assertEqual(kwargs['example'], self.data['text']) - @patch('api.views.auto_labeling.AutoLabelingConfigParameterTest.send_request', return_value={}) + @patch('auto_labeling.views.AutoLabelingConfigParameterTest.send_request', return_value={}) def test_called_with_image(self, mock): self.data['text'] = str(data_dir / 'images/1500x500.jpeg') self.assert_create(self.project.users[0], status.HTTP_200_OK) @@ -118,7 +118,7 @@ class TestAutoLabelingText(CRUDMixin): self.example = make_doc(self.project.item) self.url = reverse(viewname='auto_labeling_annotation', args=[self.project.item.id, self.example.id]) - @patch('api.views.auto_labeling.execute_pipeline', return_value=[]) + @patch('auto_labeling.views.execute_pipeline', return_value=[]) def test_text_task(self, mock): self.assert_create(self.project.users[0], status.HTTP_201_CREATED) _, kwargs = mock.call_args @@ -134,7 +134,7 @@ class TestAutoLabelingImage(CRUDMixin): self.example = make_image(self.project.item, str(filepath)) self.url = reverse(viewname='auto_labeling_annotation', args=[self.project.item.id, self.example.id]) - @patch('api.views.auto_labeling.execute_pipeline', return_value=[]) + @patch('auto_labeling.views.execute_pipeline', return_value=[]) def test_text_task(self, mock): self.assert_create(self.project.users[0], status.HTTP_201_CREATED) _, kwargs = mock.call_args diff --git a/backend/auto_labeling/urls.py b/backend/auto_labeling/urls.py new file mode 100644 index 00000000..4c08d675 --- /dev/null +++ b/backend/auto_labeling/urls.py @@ -0,0 +1,53 @@ +from django.urls import path + +from .views import (AutoLabelingConfigDetail, AutoLabelingConfigTest, AutoLabelingAnnotation, AutoLabelingMappingTest, + AutoLabelingTemplateListAPI, AutoLabelingTemplateDetailAPI, AutoLabelingConfigList, + AutoLabelingConfigParameterTest, AutoLabelingTemplateTest) + +urlpatterns = [ + path( + route='auto-labeling-templates', + view=AutoLabelingTemplateListAPI.as_view(), + name='auto_labeling_templates' + ), + path( + route='auto-labeling-templates/', + view=AutoLabelingTemplateDetailAPI.as_view(), + name='auto_labeling_template' + ), + path( + route='auto-labeling-configs', + view=AutoLabelingConfigList.as_view(), + name='auto_labeling_configs' + ), + path( + route='auto-labeling-configs/', + view=AutoLabelingConfigDetail.as_view(), + name='auto_labeling_config' + ), + path( + route='auto-labeling-config-testing', + view=AutoLabelingConfigTest.as_view(), + name='auto_labeling_config_test' + ), + path( + route='examples//auto-labeling', + view=AutoLabelingAnnotation.as_view(), + name='auto_labeling_annotation' + ), + path( + route='auto-labeling-parameter-testing', + view=AutoLabelingConfigParameterTest.as_view(), + name='auto_labeling_parameter_testing' + ), + path( + route='auto-labeling-template-testing', + view=AutoLabelingTemplateTest.as_view(), + name='auto_labeling_template_test' + ), + path( + route='auto-labeling-mapping-testing', + view=AutoLabelingMappingTest.as_view(), + name='auto_labeling_mapping_test' + ) +] diff --git a/backend/api/views/auto_labeling.py b/backend/auto_labeling/views.py similarity index 96% rename from backend/api/views/auto_labeling.py rename to backend/auto_labeling/views.py index 137c0e17..41c44c98 100644 --- a/backend/api/views/auto_labeling.py +++ b/backend/auto_labeling/views.py @@ -16,14 +16,12 @@ from rest_framework.permissions import IsAuthenticated from rest_framework.response import Response from rest_framework.views import APIView +from api.models import AutoLabelingConfig, Example, Project from members.permissions import IsInProjectOrAdmin, IsProjectAdmin - -from ..exceptions import (AutoLabelingException, AutoLabelingPermissionDenied, - AWSTokenError, SampleDataException, - TemplateMappingError, URLConnectionError) -from ..models import AutoLabelingConfig, Example, Project -from ..serializers import (AutoLabelingConfigSerializer, - get_annotation_serializer) +from .exceptions import (AutoLabelingException, AutoLabelingPermissionDenied, + AWSTokenError, SampleDataException, + TemplateMappingError, URLConnectionError) +from .serializers import (AutoLabelingConfigSerializer, get_annotation_serializer) class AutoLabelingTemplateListAPI(APIView): diff --git a/backend/members/tests.py b/backend/members/tests.py index 4ab43985..e2e91cca 100644 --- a/backend/members/tests.py +++ b/backend/members/tests.py @@ -3,7 +3,7 @@ from rest_framework import status from rest_framework.reverse import reverse from roles.models import Role -from .models import Member +from members.models import Member from api.tests.api.utils import (CRUDMixin, prepare_project, make_user) From 7c9eb290200ee5ea58c509f3f1b481ca3e96013c Mon Sep 17 00:00:00 2001 From: Hironsan Date: Thu, 20 Jan 2022 13:33:31 +0900 Subject: [PATCH 2/2] Move auto labeling model to auto labeling app --- .../0030_delete_autolabelingconfig.py | 26 +++++++++++++ backend/api/models.py | 30 --------------- backend/auto_labeling/admin.py | 2 +- .../auto_labeling/migrations/0001_initial.py | 37 +++++++++++++++++++ backend/auto_labeling/models.py | 34 +++++++++++++++++ backend/auto_labeling/serializers.py | 3 +- backend/auto_labeling/views.py | 3 +- 7 files changed, 102 insertions(+), 33 deletions(-) create mode 100644 backend/api/migrations/0030_delete_autolabelingconfig.py create mode 100644 backend/auto_labeling/migrations/0001_initial.py diff --git a/backend/api/migrations/0030_delete_autolabelingconfig.py b/backend/api/migrations/0030_delete_autolabelingconfig.py new file mode 100644 index 00000000..462d5a27 --- /dev/null +++ b/backend/api/migrations/0030_delete_autolabelingconfig.py @@ -0,0 +1,26 @@ +# Generated by Django 3.2.11 on 2022-01-20 04:17 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('api', '0029_auto_20220119_2333'), + ] + + operations = [ + migrations.SeparateDatabaseAndState( + state_operations=[ + migrations.DeleteModel( + name='AutoLabelingConfig', + ), + ], + database_operations=[ + migrations.AlterModelTable( + name='AutoLabelingConfig', + table='auto_labeling_autolabelingconfig' + ) + ] + ) + ] diff --git a/backend/api/models.py b/backend/api/models.py index 28e12a18..8ecae2b0 100644 --- a/backend/api/models.py +++ b/backend/api/models.py @@ -3,7 +3,6 @@ import random import string import uuid -from auto_labeling_pipeline.models import RequestModelFactory from django.contrib.auth.models import User from django.core.exceptions import ValidationError from django.db import models @@ -412,35 +411,6 @@ class TextLabel(Annotation): ) -class AutoLabelingConfig(models.Model): - model_name = models.CharField(max_length=100) - model_attrs = models.JSONField(default=dict) - template = models.TextField(default='') - label_mapping = models.JSONField(default=dict, blank=True) - project = models.ForeignKey( - to=Project, - on_delete=models.CASCADE, - related_name='auto_labeling_config' - ) - default = models.BooleanField(default=False) - created_at = models.DateTimeField(auto_now_add=True) - updated_at = models.DateTimeField(auto_now=True) - - def __str__(self): - return self.model_name - - def clean_fields(self, exclude=None): - super().clean_fields(exclude=exclude) - try: - RequestModelFactory.find(self.model_name) - except NameError: - message = f'The specified model name {self.model_name} does not exist.' - raise ValidationError(message) - except Exception: - message = 'The attributes does not match the model.' - raise ValidationError(message) - - class RelationTypes(models.Model): color = models.TextField() name = models.TextField() diff --git a/backend/auto_labeling/admin.py b/backend/auto_labeling/admin.py index 411e850f..7495bcf8 100644 --- a/backend/auto_labeling/admin.py +++ b/backend/auto_labeling/admin.py @@ -1,6 +1,6 @@ from django.contrib import admin -from api.models import AutoLabelingConfig +from .models import AutoLabelingConfig class AutoLabelingConfigAdmin(admin.ModelAdmin): diff --git a/backend/auto_labeling/migrations/0001_initial.py b/backend/auto_labeling/migrations/0001_initial.py new file mode 100644 index 00000000..8cfbbf27 --- /dev/null +++ b/backend/auto_labeling/migrations/0001_initial.py @@ -0,0 +1,37 @@ +# Generated by Django 3.2.11 on 2022-01-20 04:17 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ('api', '0030_delete_autolabelingconfig'), + ] + + operations = [ + migrations.SeparateDatabaseAndState( + state_operations=[ + migrations.CreateModel( + name='AutoLabelingConfig', + fields=[ + ('id', + models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('model_name', models.CharField(max_length=100)), + ('model_attrs', models.JSONField(default=dict)), + ('template', models.TextField(default='')), + ('label_mapping', models.JSONField(blank=True, default=dict)), + ('default', models.BooleanField(default=False)), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('updated_at', models.DateTimeField(auto_now=True)), + ('project', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, + related_name='auto_labeling_config', to='api.project')), + ], + ), + ], + database_operations=[] + ) + ] diff --git a/backend/auto_labeling/models.py b/backend/auto_labeling/models.py index e69de29b..502e6c7f 100644 --- a/backend/auto_labeling/models.py +++ b/backend/auto_labeling/models.py @@ -0,0 +1,34 @@ +from auto_labeling_pipeline.models import RequestModelFactory +from django.core.exceptions import ValidationError +from django.db import models + +from api.models import Project + + +class AutoLabelingConfig(models.Model): + model_name = models.CharField(max_length=100) + model_attrs = models.JSONField(default=dict) + template = models.TextField(default='') + label_mapping = models.JSONField(default=dict, blank=True) + project = models.ForeignKey( + to=Project, + on_delete=models.CASCADE, + related_name='auto_labeling_config' + ) + default = models.BooleanField(default=False) + created_at = models.DateTimeField(auto_now_add=True) + updated_at = models.DateTimeField(auto_now=True) + + def __str__(self): + return self.model_name + + def clean_fields(self, exclude=None): + super().clean_fields(exclude=exclude) + try: + RequestModelFactory.find(self.model_name) + except NameError: + message = f'The specified model name {self.model_name} does not exist.' + raise ValidationError(message) + except Exception: + message = 'The attributes does not match the model.' + raise ValidationError(message) diff --git a/backend/auto_labeling/serializers.py b/backend/auto_labeling/serializers.py index 857d7d10..4bf52d2b 100644 --- a/backend/auto_labeling/serializers.py +++ b/backend/auto_labeling/serializers.py @@ -1,9 +1,10 @@ from auto_labeling_pipeline.models import RequestModelFactory from rest_framework import serializers -from api.models import AutoLabelingConfig, DOCUMENT_CLASSIFICATION, SEQUENCE_LABELING, SEQ2SEQ, SPEECH2TEXT, \ +from api.models import DOCUMENT_CLASSIFICATION, SEQUENCE_LABELING, SEQ2SEQ, SPEECH2TEXT, \ IMAGE_CLASSIFICATION from api.serializers import CategorySerializer, SpanSerializer, TextLabelSerializer +from .models import AutoLabelingConfig class AutoLabelingConfigSerializer(serializers.ModelSerializer): diff --git a/backend/auto_labeling/views.py b/backend/auto_labeling/views.py index 41c44c98..345d2724 100644 --- a/backend/auto_labeling/views.py +++ b/backend/auto_labeling/views.py @@ -16,11 +16,12 @@ from rest_framework.permissions import IsAuthenticated from rest_framework.response import Response from rest_framework.views import APIView -from api.models import AutoLabelingConfig, Example, Project +from api.models import Example, Project from members.permissions import IsInProjectOrAdmin, IsProjectAdmin from .exceptions import (AutoLabelingException, AutoLabelingPermissionDenied, AWSTokenError, SampleDataException, TemplateMappingError, URLConnectionError) +from .models import AutoLabelingConfig from .serializers import (AutoLabelingConfigSerializer, get_annotation_serializer)