Merge pull request #248 from makeopensource/develop

Release for sprint 3

Author: jessehartloff

.dockerignore

@@ -1,4 +1,4 @@
 /**/node_modules
 /logs
-/devU-api/src/tango/tests/test_files
+/devU-api/src/autograders/tango/tests/test_files
 .env.*

.github/workflows/api.yml

@@ -42,9 +42,3 @@ jobs:
 
           docker build . -f api.Dockerfile -t $IMAGE_NAME
           docker push $IMAGE_NAME
-
-      - name: build tango docker
-        run: |
-          IMAGE_NAME=ghcr.io/${{ env.repo_url }}/tango:${{ steps.get_branch.outputs.branch_name }}
-          docker build ./tango -f ./tango/Dockerfile -t $IMAGE_NAME
-          docker push $IMAGE_NAME

.gitmodules

@@ -21,4 +21,4 @@ COPY ./devU-client/ .
 COPY --from=module_builder /tmp/devu-shared-modules ./devu-shared-modules
 
 # build frontend during run so that we can modify baseurl via docker envoirment
-CMD npm run --silent build-docker && rm -rf /out/* && cp -r /app/dist/* /out
+CMD npm run build-docker && rm -rf /out/* && cp -r /app/dist/* /out

client.Dockerfile

@@ -11,104 +11,37 @@ For now the only reason we're including docker is to more easily control the dev
 
 ## Running the Project Locally
 
-### Getting Everything Started
-
-Once you've got these installed, we can build our container and run it
-
-#### Note to run the postgres container locally using the command below
-
-You have to modify `devU-api/src/environment.ts`
-
-change
-
-`dbHost: (load('database.host') || 'localhost') as string`
-
-to
-
-`dbHost: 'localhost'`
-
-This will probably be fixed in the future but for now the above steps are necessary
-
-#### Using docker compose
-
-We use [docker compose profiles](https://docs.docker.com/compose/profiles/) to selectively start services in the main docker-compose when developing.
-
-Assuming you are in api dir `devU-api`, To start all api services except the api run
-
-```
-npm run api-services
-```
-
-To stop the services
-
-```
-npm run api-services-stop
-```
-
-Then install dependencies using
-
-```
-npm install
-```
-
-Once you've got all the dependencies installed you can run the project via
-
-```
-npm start
-```
-
-#### Manually:
-
-```
-docker run \
-  --name typeorm-postgres \
-  -p 5432:5432 \
-  -e POSTGRES_PASSWORD=password \
-  -e POSTGRES_DB=typescript_api \
-  -e POSTGRES_USER=typescript_user \
-  -d postgres
-```
-
-Install all node dependencies. All of the database environment variables can change, and can be set as environment variables on your machine if you want to overwrite the defaults
-
-```
-docker run \
-  --name minio \
-  -p 9002:9000 \
-  -p 9001:9001 \
-  -v /tmp/data:/data \
-  -e "MINIO_ROOT_USER=typescript_user" \
-  -e "MINIO_ROOT_PASSWORD=changeMe" \
-  -d minio/minio server /data --console-address ":9001"
-```
-
-Install all node dependencies. All of the database environment variables can change, and can be set as environment variables on your machine if you want to overwrite the defaults
-
-```
-npm install
-```
-
-Run the setup script to create local development auth keys. These are used in local development for signing and authenticating JWTs.
-
-```
-npm run generate-config
-```
-
-Run the initial migrations to setup our DB schema
-
-```
-npm run typeorm -- migration:run -d src/database.ts
-```
-
-Once you've got all the dependencies installed you can run the project via
-
-```
-npm start
-```
-
-By default the project runs at `localhost:3001`, but you can change the port by setting an alternate port by setting the `PORT` environment variable.
-
-If you're working in vscode, a configuration has been included to allow for debugging. Open up the VS Code Run and Debug section and click `Debug API`.
+### Quick Start
+
+The instructions below assume you are in the api dir `/devU-api/`
+
+You must have the following tools installed
+
+* Docker
+* Node >= v20
+
+Once you've got these installed,
+
+1. We use [docker compose profiles](https://docs.docker.com/compose/profiles/) 
+    to selectively start services in the main docker-compose when developing.
+    This starts all required services for the api depends (database, frontend etc.)
+    ```
+    npm run api-services
+    ```
+    
+    To remove all related containers 
+    ```
+    npm run api-services-stop
+    ```
+
+2. Install dependencies using
+    ```
+    npm install
+    ```
+3. Once you've got all the dependencies installed you can run the project via
+    ```
+    npm run start
+    ```
 
 ### Convenient Devtools
 
@@ -173,7 +106,7 @@ Here's what you need to know:
 
 Here's the basic layout of the application
 
-![control flow of the api](/docs/controlFlow.png 'Control Flow')
+![control flow of the api](./docs/controlFlow.png 'Control Flow')
 
 Let's take this from the top
 
@@ -218,9 +151,8 @@ When developing if you need to create a new type,
 
 This will update the types in `devU-api` and `devU-client` folders.
 
-**Note if the types are not being detected by your IDE**
-
-**Go to the `devU-api/` and `devU-client/` and run `npm install` in each folder to update the shared modules.**
+if the types are not being detected, Go to the `devU-api/` and `devU-client/` 
+and run `npm install` in each folder to update the shared modules.
 
 ### Testing
 
@@ -257,7 +189,7 @@ I wouldn't recommend digging that far down as the of tests should be more human-
 If the schema needs to be updated, you can do so by updating the models and running
 
 ```
-npm run typeorm migration:generate -- -d src/database src/migration/<generatedMigrationName>
+npm run create-migration someMeaningfulMigrationName
 ```
 
 Doing so will attempt to create an auto migration from any changes within the `src/models` directory and add it to `src/migrations`. If an auto migration is generated for you (always check your auto migrations), you can run it with the above migration command

devU-api/README.md

@@ -9,15 +9,16 @@
     "create-migrate": "npx typeorm-ts-node-commonjs migration:generate -d src/database.ts",
     "update-shared": "cd ../devU-shared && npm run build-local && cd ../devU-api && npm i",
     "typeorm": "typeorm-ts-node-commonjs",
+    "create-migration": "ts-node scripts/create-migration.ts",
     "test": "jest --passWithNoTests",
     "clean": "rimraf build/*",
     "lint": "tsc",
     "build": "npm-run-all clean lint",
     "format": "prettier --write \"./**/*.{js,ts,json,md}\"",
     "pre-commit": "lint-staged",
     "populate-db": "ts-node-dev ./scripts/populate-db.ts",
-    "tango": "ts-node-dev ./src/tango/tests/tango.endpoint.test.ts",
-    "api-services": "docker compose -f ../docker-compose.yml --profile dev-api up",
+    "tango": "ts-node-dev src/autograders/tango/tests/tango.endpoint.test.ts",
+    "api-services": "docker compose -f ../docker-compose.yml --profile dev-api up --build",
     "api-services-stop": "docker compose -f ../docker-compose.yml --profile dev-api stop"
   },
   "lint-staged": {
@@ -67,9 +68,11 @@
     "cors": "^2.8.5",
     "devu-shared-modules": "./devu-shared-modules",
     "express": "^4.17.1",
+    "express-list-endpoints": "^7.1.1",
     "express-validator": "^6.14.2",
     "helmet": "^4.6.0",
     "jsonwebtoken": "^9.0.2",
+    "leviathan-node-sdk": "https://gitpkg.vercel.app/makeopensource/leviathan/spec/leviathan_node?dev",
     "minio": "^8.0.0",
     "morgan": "^1.10.0",
     "multer": "^1.4.5-lts.1",

devU-api/package.json

@@ -0,0 +1,26 @@
+import { exec } from 'child_process'
+
+// Get the migration name from command line arguments
+const migrationName = process.argv[2]
+
+if (!migrationName) {
+  console.error('Migration name is required!')
+  console.error('Usage: npm run create-mig MigrationName')
+  process.exit(1)
+}
+
+const command = `npm run typeorm -- migration:generate -d src/database.ts src/migration/${migrationName}`
+console.log(`Executing: ${command}`)
+
+// Execute the command
+exec(command, (error, stdout, stderr) => {
+  if (stdout) console.log(stdout)
+  if (stderr) console.error(stderr)
+
+  if (error) {
+    console.error(`Error: ${error.message}`)
+    process.exit(1)
+  }
+
+  console.log('Migration file created successfully!')
+})

devU-api/scripts/create-migration.ts

@@ -0,0 +1,115 @@
+import {
+  CancelJobRequest,
+  createClient,
+  createConnectTransport,
+  DockerFile,
+  JobLogRequest,
+  JobService,
+  LabData,
+  LabFile,
+  LabService,
+  NewJobRequest,
+  NewLabRequest,
+  SubmissionFile,
+  UploadLabFiles,
+  UploadSubmissionFiles,
+} from 'leviathan-node-sdk'
+
+
+const leviUrl = process.env.LEVIATHAN_URL || 'http://localhost:9221'
+console.log(`Leviathan url set to ${leviUrl}`)
+
+const transport = createConnectTransport({
+  baseUrl: leviUrl,
+  httpVersion: '2',
+})
+
+const jobService = createClient(JobService, transport)
+const labService = createClient(LabService, transport)
+
+export function bufferToBlob(multerFile: Express.Multer.File): Blob {
+  return new Blob([multerFile.buffer], { type: multerFile.mimetype })
+}
+
+export async function sendSubmission(labId: bigint, submission: Array<SubmissionFile>) {
+  const fileId = await UploadSubmissionFiles(leviUrl, submission)
+  const resp = await jobService.newJob(<NewJobRequest>{
+    tmpSubmissionFolderId: fileId,
+    labID: labId,
+  })
+
+  return resp.jobId
+}
+
+
+export async function createNewLab(lab: LabData, dockerfile: DockerFile, labFiles: Array<LabFile>) {
+  const fileId = await UploadLabFiles(leviUrl, dockerfile, labFiles)
+  const resp = await labService.newLab(<NewLabRequest>{
+    labData: lab,
+    tmpFolderId: fileId,
+  })
+
+  return resp.labId
+}
+
+/**
+ * streams job status,
+ * the stream will exit on its own once the job is done, can be cancelled by calling controller.abort()
+ * @returns a stream and a controller can be used to cancel the stream
+ * @see waitForJob - for usage example
+ */
+export function streamJob(jobId: string) {
+  const controller = new AbortController()
+  const dataStream = jobService.streamStatus(
+    <JobLogRequest>{ jobId },
+    { signal: controller.signal },
+  )
+
+  return { dataStream, controller }
+}
+
+
+/**
+ * gets current job status with logs
+ */
+export async function getStatus(jobId: string) {
+  const resp = await jobService.getStatus(<JobLogRequest>{ jobId })
+  // strip out grpc metadata
+  const { $unknown, $typeName, ...info } = resp.jobInfo!
+  const logs = resp.logs
+  return { info, logs }
+}
+
+/**
+ * Blocks until job is complete
+ * @see streamJob - is used under the hood
+ */
+export async function waitForJob(jobId: string) {
+  const { dataStream } = streamJob(jobId)
+
+  let jobInfo: { jobId: string; status: string; statusMessage: string } = {
+    jobId: '',
+    status: '',
+    statusMessage: '',
+  }
+  let logs: string = ''
+
+  for await (const chunk of dataStream) {
+    if (!chunk.jobInfo) {
+      console.warn('Empty job state')
+      continue
+    }
+
+    const { $unknown, $typeName, ...rest } = chunk.jobInfo!
+    console.debug('Job', rest)
+
+    jobInfo = rest
+    logs = chunk.logs
+  }
+
+  return { jobInfo, logs }
+}
+
+export async function cancelJob(jobId: string) {
+  await jobService.cancelJob(<CancelJobRequest>{ jobId })
+}

devU-api/src/autograders/leviathan.service.ts

@@ -66,11 +66,12 @@ export async function listByCourse(courseId: number) {
 export async function listByCourseReleased(courseId: number) {
   // TODO: filter by start date after current time
   // const now = new Date(Date.now())
-  const allAssignments = await connect().findBy({ courseId: courseId, /*startDate: MoreThanOrEqual(now),*/ deletedAt: IsNull() })
-
   // console.log("ASSIGNMENTS WITH FILTER: ", allAssignments)
 
-  return allAssignments;
+  return await connect().findBy({
+    courseId: courseId, /*startDate: MoreThanOrEqual(now),*/
+    deletedAt: IsNull(),
+  })
 }
 
 export async function isReleased(id: number) {
@@ -87,7 +88,10 @@ export async function isReleased(id: number) {
 }
 
 async function getMaxSubmissionsAndDeadline(id: number) {
-  return await connect().findOne({ where: { id: id, deletedAt: IsNull() }, select: ['maxSubmissions', 'maxFileSize', 'disableHandins', 'endDate'] })
+  return await connect().findOne({
+    where: { id: id, deletedAt: IsNull() },
+    select: ['maxSubmissions', 'maxFileSize', 'disableHandins', 'endDate'],
+  })
 }
 
 async function processFiles(req: Request) {