diff --git a/app/app/settings.py b/app/app/settings.py index f0de907c..e634230d 100644 --- a/app/app/settings.py +++ b/app/app/settings.py @@ -57,6 +57,15 @@ INSTALLED_APPS = [ 'webpack_loader', ] +CLOUD_BROWSER_APACHE_LIBCLOUD_PROVIDER = env('CLOUD_BROWSER_LIBCLOUD_PROVIDER', None) +CLOUD_BROWSER_APACHE_LIBCLOUD_ACCOUNT = env('CLOUD_BROWSER_LIBCLOUD_ACCOUNT', None) +CLOUD_BROWSER_APACHE_LIBCLOUD_SECRET_KEY = env('CLOUD_BROWSER_LIBCLOUD_KEY', None) + +if CLOUD_BROWSER_APACHE_LIBCLOUD_PROVIDER: + CLOUD_BROWSER_DATASTORE = 'ApacheLibcloud' + CLOUD_BROWSER_OBJECT_REDIRECT_URL = '/v1/cloud-upload' + INSTALLED_APPS.append('cloud_browser') + MIDDLEWARE = [ 'django.middleware.security.SecurityMiddleware', 'whitenoise.middleware.WhiteNoiseMiddleware', diff --git a/app/app/urls.py b/app/app/urls.py index e9bc2ea4..0fa1ed36 100644 --- a/app/app/urls.py +++ b/app/app/urls.py @@ -13,6 +13,7 @@ Including another URLconf 1. Import the include() function: from django.urls import include, path 2. Add a URL to urlpatterns: path('blog/', include('blog.urls')) """ +from django.conf import settings from django.contrib import admin from django.urls import path, include, re_path from django.contrib.auth.views import PasswordResetView, LogoutView @@ -29,3 +30,6 @@ urlpatterns = [ path('api-auth/', include('rest_framework.urls')), path('v1/', include('server.api_urls')), ] + +if 'cloud_browser' in settings.INSTALLED_APPS: + urlpatterns.append(path('cloud-storage/', include('cloud_browser.urls'))) diff --git a/app/server/api.py b/app/server/api.py index bc853c4d..93d8c0e4 100644 --- a/app/server/api.py +++ b/app/server/api.py @@ -1,8 +1,11 @@ from collections import Counter -from django.shortcuts import get_object_or_404 +from django.conf import settings +from django.shortcuts import get_object_or_404, redirect from django_filters.rest_framework import DjangoFilterBackend from django.db.models import Count +from libcloud.base import DriverType, get_driver +from libcloud.storage.types import ContainerDoesNotExistError, ObjectDoesNotExistError from rest_framework import generics, filters, status from rest_framework.exceptions import ParseError, ValidationError from rest_framework.permissions import IsAuthenticated, IsAdminUser @@ -16,7 +19,7 @@ from .models import Project, Label, Document from .permissions import IsAdminUserAndWriteOnly, IsProjectUser, IsOwnAnnotation from .serializers import ProjectSerializer, LabelSerializer, DocumentSerializer, UserSerializer from .serializers import ProjectPolymorphicSerializer -from .utils import CSVParser, JSONParser, PlainTextParser, CoNLLParser +from .utils import CSVParser, JSONParser, PlainTextParser, CoNLLParser, iterable_to_io from .utils import JSONLRenderer from .utils import JSONPainter, CSVPainter @@ -29,6 +32,15 @@ class Me(APIView): return Response(serializer.data) +class Features(APIView): + permission_classes = (IsAuthenticated,) + + def get(self, request, *args, **kwargs): + return Response({ + 'cloud_upload': bool(settings.CLOUD_BROWSER_APACHE_LIBCLOUD_PROVIDER), + }) + + class ProjectList(generics.ListCreateAPIView): queryset = Project.objects.all() serializer_class = ProjectPolymorphicSerializer @@ -180,24 +192,87 @@ class TextUploadAPI(APIView): def post(self, request, *args, **kwargs): if 'file' not in request.data: raise ParseError('Empty content') - project = get_object_or_404(Project, pk=self.kwargs['project_id']) - parser = self.select_parser(request.data['format']) - data = parser.parse(request.data['file']) - storage = project.get_storage(data) - storage.save(self.request.user) + + self.save_file( + user=request.user, + file=request.data['file'], + file_format=request.data['format'], + project_id=kwargs['project_id'], + ) + return Response(status=status.HTTP_201_CREATED) - def select_parser(self, format): - if format == 'plain': + @classmethod + def save_file(cls, user, file, file_format, project_id): + project = get_object_or_404(Project, pk=project_id) + parser = cls.select_parser(file_format) + data = parser.parse(file) + storage = project.get_storage(data) + storage.save(user) + + @classmethod + def select_parser(cls, file_format): + if file_format == 'plain': return PlainTextParser() - elif format == 'csv': + elif file_format == 'csv': return CSVParser() - elif format == 'json': + elif file_format == 'json': return JSONParser() - elif format == 'conll': + elif file_format == 'conll': return CoNLLParser() else: - raise ValidationError('format {} is invalid.'.format(format)) + raise ValidationError('format {} is invalid.'.format(file_format)) + + +class CloudUploadAPI(APIView): + permission_classes = TextUploadAPI.permission_classes + + def get(self, request, *args, **kwargs): + try: + project_id = request.query_params['project_id'] + file_format = request.query_params['upload_format'] + cloud_container = request.query_params['container'] + cloud_object = request.query_params['object'] + except KeyError as ex: + raise ValidationError('query parameter {} is missing'.format(ex)) + + try: + cloud_file = self.get_cloud_object_as_io(cloud_container, cloud_object) + except ContainerDoesNotExistError: + raise ValidationError('cloud container {} does not exist'.format(cloud_container)) + except ObjectDoesNotExistError: + raise ValidationError('cloud object {} does not exist'.format(cloud_object)) + + TextUploadAPI.save_file( + user=request.user, + file=cloud_file, + file_format=file_format, + project_id=project_id, + ) + + next_url = request.query_params.get('next') + + if next_url == 'about:blank': + return Response(data='', content_type='text/plain', status=status.HTTP_201_CREATED) + + if next_url: + return redirect(next_url) + + return Response(status=status.HTTP_201_CREATED) + + @classmethod + def get_cloud_object_as_io(cls, container_name, object_name): + provider = settings.CLOUD_BROWSER_APACHE_LIBCLOUD_PROVIDER.lower() + account = settings.CLOUD_BROWSER_APACHE_LIBCLOUD_ACCOUNT + key = settings.CLOUD_BROWSER_APACHE_LIBCLOUD_SECRET_KEY + + driver = get_driver(DriverType.STORAGE, provider) + client = driver(account, key) + + cloud_container = client.get_container(container_name) + cloud_object = cloud_container.get_object(object_name) + + return iterable_to_io(cloud_object.as_stream()) class TextDownloadAPI(APIView): diff --git a/app/server/api_urls.py b/app/server/api_urls.py index 15f894d9..ffeeb0c8 100644 --- a/app/server/api_urls.py +++ b/app/server/api_urls.py @@ -1,17 +1,19 @@ from django.urls import path from rest_framework.urlpatterns import format_suffix_patterns -from .api import Me +from .api import Me, Features from .api import ProjectList, ProjectDetail from .api import LabelList, LabelDetail from .api import DocumentList, DocumentDetail from .api import AnnotationList, AnnotationDetail -from .api import TextUploadAPI, TextDownloadAPI +from .api import TextUploadAPI, TextDownloadAPI, CloudUploadAPI from .api import StatisticsAPI urlpatterns = [ path('me', Me.as_view(), name='me'), + path('features', Features.as_view(), name='features'), + path('cloud-upload', CloudUploadAPI.as_view(), name='cloud_uploader'), path('projects', ProjectList.as_view(), name='project_list'), path('projects/', ProjectDetail.as_view(), name='project_detail'), path('projects//statistics', diff --git a/app/server/permissions.py b/app/server/permissions.py index e82a3416..99f7bc58 100644 --- a/app/server/permissions.py +++ b/app/server/permissions.py @@ -9,7 +9,7 @@ class IsProjectUser(BasePermission): def has_permission(self, request, view): user = request.user - project_id = view.kwargs.get('project_id') + project_id = view.kwargs.get('project_id') or request.query_params.get('project_id') project = get_object_or_404(Project, pk=project_id) return user in project.users.all() diff --git a/app/server/static/components/http.js b/app/server/static/components/http.js index 4124ec9e..df1fa1d3 100644 --- a/app/server/static/components/http.js +++ b/app/server/static/components/http.js @@ -7,4 +7,5 @@ const HTTP = axios.create({ baseURL: `/v1/${baseUrl}`, }); +export const newHttpClient = axios.create; export default HTTP; diff --git a/app/server/static/components/mixin.js b/app/server/static/components/mixin.js index d7ac857c..7f006a04 100644 --- a/app/server/static/components/mixin.js +++ b/app/server/static/components/mixin.js @@ -2,7 +2,7 @@ import * as marked from 'marked'; import hljs from 'highlight.js'; import VueJsonPretty from 'vue-json-pretty'; import isEmpty from 'lodash.isempty'; -import HTTP from './http'; +import HTTP, { newHttpClient } from './http'; import Messages from './messages.vue'; const getOffsetFromUrl = (url) => { @@ -227,13 +227,48 @@ export const uploadMixin = { messages: [], format: 'json', isLoading: false, + isCloudUploadActive: false, + canUploadFromCloud: false, }), mounted() { hljs.initHighlighting(); }, + created() { + newHttpClient().get('/v1/features').then((response) => { + this.canUploadFromCloud = response.data.cloud_upload; + }); + }, + + computed: { + projectId() { + return window.location.pathname.split('/')[2]; + }, + + postUploadUrl() { + return window.location.pathname.split('/').slice(0, -1).join('/'); + }, + + cloudUploadUrl() { + return '/cloud-storage' + + `?project_id=${this.projectId}` + + `&upload_format=${this.format}` + + `&next=${encodeURIComponent('about:blank')}`; + }, + }, + methods: { + cloudUpload() { + const iframeUrl = this.$refs.cloudUploadPane.contentWindow.location.href; + if (iframeUrl.indexOf('/v1/cloud-upload') > -1) { + this.isCloudUploadActive = false; + this.$nextTick(() => { + window.location.href = this.postUploadUrl; + }); + } + }, + upload() { this.isLoading = true; this.file = this.$refs.file.files[0]; @@ -250,7 +285,7 @@ export const uploadMixin = { .then((response) => { console.log(response); // eslint-disable-line no-console this.messages = []; - window.location = window.location.pathname.split('/').slice(0, -1).join('/'); + window.location = this.postUploadUrl; }) .catch((error) => { this.isLoading = false; diff --git a/app/server/static/components/projects.vue b/app/server/static/components/projects.vue index ebc39c86..b527a4a9 100644 --- a/app/server/static/components/projects.vue +++ b/app/server/static/components/projects.vue @@ -108,12 +108,11 @@