From 3f90b3e855e4a4b89af8bfd9cb4b1514eaa9d9ac Mon Sep 17 00:00:00 2001
From: Hironsan <light.tree.1.13@gmail.com>
Date: Mon, 4 Jul 2022 12:02:18 +0900
Subject: [PATCH] Support bounding box

---
 frontend/domain/models/tasks/boundingBox.ts   |  43 +++
 .../projects/_id/object-detection/index.vue   | 319 ++++++++++++++++++
 frontend/plugins/services.ts                  |   8 +-
 .../boundingBox/apiBoundingBoxRepository.ts   |  40 +++
 .../boundingBoxApplicationService.ts          |  55 +++
 .../tasks/boundingBox/boundingBoxData.ts      |  21 ++
 6 files changed, 485 insertions(+), 1 deletion(-)
 create mode 100644 frontend/domain/models/tasks/boundingBox.ts
 create mode 100644 frontend/pages/projects/_id/object-detection/index.vue
 create mode 100644 frontend/repositories/tasks/boundingBox/apiBoundingBoxRepository.ts
 create mode 100644 frontend/services/application/tasks/boundingBox/boundingBoxApplicationService.ts
 create mode 100644 frontend/services/application/tasks/boundingBox/boundingBoxData.ts

diff --git a/frontend/domain/models/tasks/boundingBox.ts b/frontend/domain/models/tasks/boundingBox.ts
new file mode 100644
index 00000000..d6b737ed
--- /dev/null
+++ b/frontend/domain/models/tasks/boundingBox.ts
@@ -0,0 +1,43 @@
+export class BoundingBoxItem {
+  constructor(
+    public id: number,
+    public uuid: string,
+    public label: number,
+    public x: number,
+    public y: number,
+    public width: number,
+    public height: number
+  ) {}
+
+  static valueOf({
+    id,
+    uuid,
+    label,
+    x,
+    y,
+    width,
+    height
+  }: {
+    id: number
+    uuid: string
+    label: number
+    x: number
+    y: number
+    width: number
+    height: number
+  }): BoundingBoxItem {
+    return new BoundingBoxItem(id, uuid, label, x, y, width, height)
+  }
+
+  toObject(): Object {
+    return {
+      id: this.id,
+      uuid: this.uuid,
+      label: this.label,
+      x: this.x,
+      y: this.y,
+      width: this.width,
+      height: this.height
+    }
+  }
+}
diff --git a/frontend/pages/projects/_id/object-detection/index.vue b/frontend/pages/projects/_id/object-detection/index.vue
new file mode 100644
index 00000000..0f721735
--- /dev/null
+++ b/frontend/pages/projects/_id/object-detection/index.vue
@@ -0,0 +1,319 @@
+<template>
+  <layout-text v-if="image.id">
+    <template #header>
+      <toolbar-laptop
+        :doc-id="image.id"
+        :enable-auto-labeling.sync="enableAutoLabeling"
+        :guideline-text="project.guideline"
+        :is-reviewd="image.isConfirmed"
+        :total="images.count"
+        class="d-none d-sm-block"
+        @click:clear-label="clear"
+        @click:review="confirm"
+      >
+        <v-btn-toggle v-model="labelOption" mandatory class="ms-2">
+          <v-btn icon>
+            <v-icon>{{ mdiFormatListBulleted }}</v-icon>
+          </v-btn>
+          <v-btn icon>
+            <v-icon>{{ mdiText }}</v-icon>
+          </v-btn>
+        </v-btn-toggle>
+      </toolbar-laptop>
+      <toolbar-mobile :total="images.count" class="d-flex d-sm-none" />
+    </template>
+    <template #content>
+      <v-card>
+        <v-card-title>
+          <v-chip-group v-model="selectedLabelIndex" column>
+            <v-chip
+              v-for="item in labels"
+              :key="item.id"
+              :color="item.backgroundColor"
+              filter
+              :text-color="$contrastColor(item.backgroundColor)"
+            >
+              {{ item.text }}
+              <v-avatar
+                v-if="item.suffixKey"
+                right
+                color="white"
+                class="black--text font-weight-bold"
+              >
+                {{ item.suffixKey }}
+              </v-avatar>
+            </v-chip>
+          </v-chip-group>
+        </v-card-title>
+        <v-divider />
+        <v-bounding-box
+          :rectangles="filteredRegions"
+          :highlight-id="highlightId"
+          :image-url="image.fileUrl"
+          :labels="bboxLabels"
+          :selected-label="selectedLabel"
+          :scale="scale"
+          @add-rectangle="add"
+          @update-rectangle="update"
+          @delete-rectangle="remove"
+          @update-scale="updateScale"
+          @select-rectangle="selectRegion"
+        />
+      </v-card>
+    </template>
+    <template #sidebar>
+      <annotation-progress :progress="progress" />
+      <list-metadata :metadata="image.meta" class="mt-4" />
+      <region-list
+        v-if="annotations.length > 0"
+        class="mt-4"
+        :regions="regionList"
+        @change-visibility="changeVisibility"
+        @hover-region="hoverRegion"
+        @unhover-region="unhoverRegion"
+      />
+    </template>
+  </layout-text>
+</template>
+
+<script>
+import _ from 'lodash'
+import { mdiText, mdiFormatListBulleted } from '@mdi/js'
+import { toRefs, useContext } from '@nuxtjs/composition-api'
+import LayoutText from '@/components/tasks/layout/LayoutText'
+import ListMetadata from '@/components/tasks/metadata/ListMetadata'
+import ToolbarLaptop from '@/components/tasks/toolbar/ToolbarLaptop'
+import ToolbarMobile from '@/components/tasks/toolbar/ToolbarMobile'
+import { useLabelList } from '@/composables/useLabelList'
+import AnnotationProgress from '@/components/tasks/sidebar/AnnotationProgress.vue'
+import { VBoundingBox } from 'vue-image-annotator'
+import RegionList from '@/components/tasks/image/RegionList.vue'
+
+export default {
+  components: {
+    AnnotationProgress,
+    LayoutText,
+    ListMetadata,
+    RegionList,
+    ToolbarLaptop,
+    ToolbarMobile,
+    VBoundingBox
+  },
+  layout: 'workspace',
+
+  validate({ params, query }) {
+    return /^\d+$/.test(params.id) && /^\d+$/.test(query.page)
+  },
+
+  setup() {
+    const { app } = useContext()
+    const { state, getLabelList, shortKeys } = useLabelList(app.$services.categoryType)
+
+    return {
+      ...toRefs(state),
+      getLabelList,
+      shortKeys
+    }
+  },
+
+  data() {
+    return {
+      annotations: [],
+      images: [],
+      project: {},
+      enableAutoLabeling: false,
+      labelOption: 0,
+      mdiText,
+      mdiFormatListBulleted,
+      progress: {},
+      highlightId: null,
+      selectedLabelIndex: undefined,
+      selectedRegion: undefined,
+      visibilities: {},
+      scale: 1
+    }
+  },
+
+  async fetch() {
+    this.images = await this.$services.example.fetchOne(
+      this.projectId,
+      this.$route.query.page,
+      this.$route.query.q,
+      this.$route.query.isChecked
+    )
+    const image = this.images.items[0]
+    if (this.enableAutoLabeling) {
+      await this.autoLabel(image.id)
+    }
+    await this.list(image.id)
+  },
+
+  computed: {
+    projectId() {
+      return this.$route.params.id
+    },
+
+    image() {
+      if (_.isEmpty(this.images) || this.images.items.length === 0) {
+        return {}
+      } else {
+        return this.images.items[0]
+      }
+    },
+
+    bboxLabels() {
+      return this.labels.map((label) => {
+        return {
+          id: label.id,
+          name: label.text,
+          color: label.backgroundColor
+        }
+      })
+    },
+
+    selectedLabel() {
+      if (this.selectedLabelIndex !== undefined) {
+        return this.labels[this.selectedLabelIndex]
+      } else {
+        return undefined
+      }
+    },
+
+    regionList() {
+      return this.annotations.map((annotation) => {
+        return {
+          id: annotation.uuid,
+          category: this.labels.find((label) => annotation.label === label.id).text,
+          color: this.labels.find((label) => annotation.label === label.id).backgroundColor,
+          visibility:
+            annotation.uuid in this.visibilities ? this.visibilities[annotation.uuid] : true
+        }
+      })
+    },
+
+    filteredRegions() {
+      return this.annotations
+        .filter((annotation) => this.visibilities[annotation.uuid] !== false)
+        .map((a) => {
+          return {
+            ...a,
+            id: a.uuid
+          }
+        })
+    }
+  },
+
+  watch: {
+    '$route.query': '$fetch',
+    enableAutoLabeling(val) {
+      if (val) {
+        this.list(this.image.id)
+      }
+    },
+
+    async selectedLabel(newLabel) {
+      if (newLabel !== undefined && !!this.selectedRegion) {
+        this.selectedRegion.label = newLabel.id
+        await this.$services.bbox.update(
+          this.projectId,
+          this.image.id,
+          this.selectedRegion.id,
+          this.selectedRegion
+        )
+        await this.list(this.image.id)
+      }
+    }
+  },
+
+  async created() {
+    this.getLabelList(this.projectId)
+    this.project = await this.$services.project.findById(this.projectId)
+    this.progress = await this.$services.metrics.fetchMyProgress(this.projectId)
+  },
+
+  methods: {
+    async list(imageId) {
+      this.annotations = await this.$services.bbox.list(this.projectId, imageId)
+    },
+
+    async remove(id) {
+      delete this.visibilities[id]
+      const bbox = this.annotations.find((a) => a.uuid === region.id)
+      await this.$services.bbox.delete(this.projectId, this.image.id, bbox.id)
+      await this.list(this.image.id)
+    },
+
+    async add(region) {
+      this.visibilities[region.id] = true
+      await this.$services.bbox.create(
+        this.projectId,
+        this.image.id,
+        region.id,
+        region.label,
+        region.x,
+        region.y,
+        region.width,
+        region.height
+      )
+      await this.list(this.image.id)
+    },
+
+    async update(region) {
+      const bbox = this.annotations.find((a) => a.uuid === region.id)
+      await this.$services.bbox.update(this.projectId, this.image.id, bbox.id, region)
+      await this.list(this.image.id)
+    },
+
+    changeVisibility(regionId, visibility) {
+      this.$set(this.visibilities, regionId, visibility)
+      this.visibilities = Object.assign({}, this.visibilities)
+    },
+
+    async clear() {
+      await this.$services.bbox.clear(this.projectId, this.image.id)
+      await this.list(this.image.id)
+    },
+
+    async autoLabel(imageId) {
+      try {
+        await this.$services.bbox.autoLabel(this.projectId, imageId)
+      } catch (e) {
+        console.log(e.response.data.detail)
+      }
+    },
+
+    async updateProgress() {
+      this.progress = await this.$services.metrics.fetchMyProgress(this.projectId)
+    },
+
+    async confirm() {
+      await this.$services.example.confirm(this.projectId, this.image.id)
+      await this.$fetch()
+      this.updateProgress()
+    },
+
+    hoverRegion(regionId) {
+      this.highlightId = regionId
+    },
+
+    unhoverRegion() {
+      this.highlightId = null
+    },
+
+    selectRegion(regionId) {
+      this.selectedRegion = this.annotations.find((r) => r.uuid === regionId)
+      this.selectedLabelIndex = this.labels.findIndex((l) => l.id === this.selectedRegion.label)
+    },
+
+    updateScale(scale) {
+      this.scale = scale
+    }
+  }
+}
+</script>
+
+<style scoped>
+.text-pre-wrap {
+  white-space: pre-wrap !important;
+}
+</style>
diff --git a/frontend/plugins/services.ts b/frontend/plugins/services.ts
index b320b20a..4e6d3358 100644
--- a/frontend/plugins/services.ts
+++ b/frontend/plugins/services.ts
@@ -42,6 +42,8 @@ import { DownloadFormatApplicationService } from '~/services/application/downloa
 import { APITagRepository } from '~/repositories/tag/apiTagRepository'
 import { TagApplicationService } from '~/services/application/tag/tagApplicationService'
 import { ApiRelationRepository } from '~/repositories/tasks/sequenceLabeling/apiRelationRepository'
+import { ApiBoundingBoxRepository } from '~/repositories/tasks/boundingBox/apiBoundingBoxRepository'
+import { BoundingBoxApplicationService } from '~/services/application/tasks/boundingBox/boundingBoxApplicationService'
 
 export interface Services {
   categoryType: LabelApplicationService
@@ -67,6 +69,7 @@ export interface Services {
   downloadFormat: DownloadFormatApplicationService
   download: DownloadApplicationService
   tag: TagApplicationService
+  bbox: BoundingBoxApplicationService
 }
 
 declare module 'vue/types/vue' {
@@ -97,6 +100,7 @@ const plugin: Plugin = (_, inject) => {
   const taskStatusRepository = new APITaskStatusRepository()
   const downloadFormatRepository = new APIDownloadFormatRepository()
   const downloadRepository = new APIDownloadRepository()
+  const boundingBoxRepository = new ApiBoundingBoxRepository()
 
   const categoryType = new LabelApplicationService(new APILabelRepository('category-type'))
   const spanType = new LabelApplicationService(new APILabelRepository('span-type'))
@@ -113,6 +117,7 @@ const plugin: Plugin = (_, inject) => {
     sequenceLabelingRepository,
     linkRepository
   )
+  const bbox = new BoundingBoxApplicationService(boundingBoxRepository)
   const seq2seq = new Seq2seqApplicationService(seq2seqRepository)
   const option = new OptionApplicationService(optionRepository)
   const config = new ConfigApplicationService(configRepository)
@@ -148,7 +153,8 @@ const plugin: Plugin = (_, inject) => {
     taskStatus,
     downloadFormat,
     download,
-    tag
+    tag,
+    bbox
   }
   inject('services', services)
 }
diff --git a/frontend/repositories/tasks/boundingBox/apiBoundingBoxRepository.ts b/frontend/repositories/tasks/boundingBox/apiBoundingBoxRepository.ts
new file mode 100644
index 00000000..7fc74660
--- /dev/null
+++ b/frontend/repositories/tasks/boundingBox/apiBoundingBoxRepository.ts
@@ -0,0 +1,40 @@
+import { AnnotationRepository } from '@/domain/models/tasks/annotationRepository'
+import { BoundingBoxItem } from '~/domain/models/tasks/boundingBox'
+
+export class ApiBoundingBoxRepository extends AnnotationRepository<BoundingBoxItem> {
+  constructor() {
+    super(BoundingBoxItem)
+  }
+
+  async list(projectId: string, exampleId: number): Promise<BoundingBoxItem[]> {
+    const url = `/projects/${projectId}/examples/${exampleId}/bboxes`
+    const response = await this.request.get(url)
+    return response.data.map((box: any) => BoundingBoxItem.valueOf(box))
+  }
+
+  async create(projectId: string, exampleId: number, item: BoundingBoxItem): Promise<void> {
+    const url = `/projects/${projectId}/examples/${exampleId}/bboxes`
+    await this.request.post(url, item.toObject())
+  }
+
+  async update(
+    projectId: string,
+    exampleId: number,
+    boxId: number,
+    item: BoundingBoxItem
+  ): Promise<BoundingBoxItem> {
+    const url = `/projects/${projectId}/examples/${exampleId}/bboxes/${boxId}`
+    const response = await this.request.patch(url, item.toObject())
+    return BoundingBoxItem.valueOf(response.data)
+  }
+
+  async delete(projectId: string, exampleId: number, boxId: number): Promise<void> {
+    const url = `/projects/${projectId}/examples/${exampleId}/bboxes/${boxId}`
+    await this.request.delete(url)
+  }
+
+  async bulkDelete(projectId: string, exampleId: number, boxIds: number[]): Promise<void> {
+    const url = `/projects/${projectId}/examples/${exampleId}/bboxes`
+    await this.request.delete(url, { ids: boxIds })
+  }
+}
diff --git a/frontend/services/application/tasks/boundingBox/boundingBoxApplicationService.ts b/frontend/services/application/tasks/boundingBox/boundingBoxApplicationService.ts
new file mode 100644
index 00000000..ed2b6d97
--- /dev/null
+++ b/frontend/services/application/tasks/boundingBox/boundingBoxApplicationService.ts
@@ -0,0 +1,55 @@
+import { AnnotationApplicationService } from '../annotationApplicationService'
+import { BoundingBoxDTO } from './boundingBoxData'
+import { ApiBoundingBoxRepository } from '~/repositories/tasks/boundingBox/apiBoundingBoxRepository'
+import { BoundingBoxItem } from '~/domain/models/tasks/boundingBox'
+
+export class BoundingBoxApplicationService extends AnnotationApplicationService<BoundingBoxItem> {
+  constructor(readonly repository: ApiBoundingBoxRepository) {
+    super(new ApiBoundingBoxRepository())
+  }
+
+  public async list(projectId: string, exampleId: number): Promise<BoundingBoxDTO[]> {
+    const items = await this.repository.list(projectId, exampleId)
+    return items.map((item) => new BoundingBoxDTO(item))
+  }
+
+  public async create(
+    projectId: string,
+    exampleId: number,
+    uuid: string,
+    label: number,
+    x: number,
+    y: number,
+    width: number,
+    height: number
+  ): Promise<void> {
+    const item = new BoundingBoxItem(0, uuid, label, x, y, width, height)
+    try {
+      await this.repository.create(projectId, exampleId, item)
+    } catch (e: any) {
+      console.log(e.response.data.detail)
+    }
+  }
+
+  public async update(
+    projectId: string,
+    exampleId: number,
+    annotationId: number,
+    item: BoundingBoxDTO
+  ): Promise<void> {
+    const bbox = new BoundingBoxItem(
+      item.id,
+      item.uuid,
+      item.label,
+      item.x,
+      item.y,
+      item.width,
+      item.height
+    )
+    try {
+      await this.repository.update(projectId, exampleId, annotationId, bbox)
+    } catch (e: any) {
+      console.log(e.response.data.detail)
+    }
+  }
+}
diff --git a/frontend/services/application/tasks/boundingBox/boundingBoxData.ts b/frontend/services/application/tasks/boundingBox/boundingBoxData.ts
new file mode 100644
index 00000000..c4b29890
--- /dev/null
+++ b/frontend/services/application/tasks/boundingBox/boundingBoxData.ts
@@ -0,0 +1,21 @@
+import { BoundingBoxItem } from '~/domain/models/tasks/boundingBox'
+
+export class BoundingBoxDTO {
+  id: number
+  uuid: string
+  label: number
+  x: number
+  y: number
+  width: number
+  height: number
+
+  constructor(item: BoundingBoxItem) {
+    this.id = item.id
+    this.uuid = item.uuid
+    this.label = item.label
+    this.x = item.x
+    this.y = item.y
+    this.width = item.width
+    this.height = item.height
+  }
+}