Merge pull request #10 from JBB0807/feature/deployment-yoshi

Feature/deployment yoshi

assignment-service/fly.toml

@@ -10,13 +10,20 @@ primary_region = 'sea'
 
 [env]
   PORT = '8080'
+  FLY_ORG="personal"
+  COMMON_BUCKET="snakeapi-deployment-test-bucket"
+  AWS_ACCESS_KEY_ID="tid__NSmOVaGknqitaCySppZjqVTgJSdDFnFbWcQllkC_juHwkbQZO"
+  AWS_SECRET_ACCESS_KEY="tsec_6Bz1aMbfYQftuq5WfIVEDZkHwskU4MMjVywdtxSP6uxetEBvkSC2VHI9HfTeDgHr4D6kiz"
+  AWS_ENDPOINT_URL_S3="https://fly.storage.tigris.dev"
+  AWS_REGION="auto"
+  FLY_API_BASE_URL = "https://api.machines.dev/v1"
 
 [http_service]
   internal_port = 8080
   force_https = true
   auto_stop_machines = 'stop'
   auto_start_machines = true
-  min_machines_running = 0
+  min_machines_running = 1
   processes = ['app']
 
 [[services]]

assignment-service/package.json

@@ -10,7 +10,8 @@
   "license": "ISC",
   "description": "",
   "dependencies": {
-    "axios": "^1.8.4",
+    "aws-sdk": "^2.1692.0",
+    "axios": "^1.9.0",
     "cors": "^2.8.5",
     "dotenv": "^16.5.0",
     "express": "^5.1.0",

assignment-service/server.js

@@ -4,14 +4,22 @@ const passport = require("passport");
 const session = require("express-session");
 
 const express = require("express");
+const AWS = require("aws-sdk");
 const instructorRouter = require("./routes/InstructorRouter");
 const studentRouter = require("./routes/StudentRouter");
 
+const s3 = new AWS.S3({
+  endpoint: process.env.AWS_ENDPOINT_URL_S3,
+  accessKeyId: process.env.AWS_ACCESS_KEY_ID,
+  secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY,
+  region: process.env.AWS_REGION,
+  s3ForcePathStyle: true
+});
+const BUCKET = process.env.COMMON_BUCKET;
+
 const app = express();
 app.use(express.json());
 app.use(express.urlencoded({ extended: true }));
-
-// Allow all origins (not recommended for production)
 app.use(cors());
 
 // app.use(
@@ -34,10 +42,33 @@ app.use(cors());
 //         methods: ["GET", "POST"],
 //         credentials: true,
 //     })
-// )
+// );
 
 app.use("/instructor", instructorRouter);
 app.use("/student", studentRouter);
 
+app.get("/", (req, res) => {
+    res.send("OK");
+  });
+
+app.get("/notebook/:appName", async (req, res) => {
+  try {
+    const { appName } = req.params;
+    const prefix = `${appName}/notebooks/`;
+    const list = await s3.listObjectsV2({ Bucket: BUCKET, Prefix: prefix }).promise();
+    if (!list.Contents || list.Contents.length === 0) {
+      return res.status(404).json({ error: "Notebook not found" });
+    }
+    const latest = list.Contents.reduce((prev, curr) =>
+      prev.LastModified > curr.LastModified ? prev : curr
+    );
+    const data = await s3.getObject({ Bucket: BUCKET, Key: latest.Key }).promise();
+    res.send(data.Body.toString("utf-8"));
+  } catch (error) {
+    console.error("Failed to load notebook:", error);
+    res.status(500).json({ error: "Failed to load notebook" });
+  }
+});
+
 const port = process.env.PORT || 8080;
-app.listen(port, () => console.log(`Listening on port ${port}...`));
+app.listen(port, "0.0.0.0", () => console.log(`Listening on 0.0.0.0:${port}...`));

deployment-service/Dockerfile

@@ -0,0 +1,13 @@
+FROM node:18-alpine
+WORKDIR /app
+
+COPY package.json package-lock.json ./
+RUN npm ci --only=production
+
+COPY src ./src
+COPY snakeapi_service ./snakeapi_service
+
+ENV FLY_ACCESS_TOKEN="FlyV1 fm2_lJPECAAAAAAACJJHxBByW6wRXXxQ17OD8xlRRR5cwrVodHRwczovL2FwaS5mbHkuaW8vdjGUAJLOAA//nh8Lk7lodHRwczovL2FwaS5mbHkuaW8vYWFhL3YxxDwQBQg0Vif1OLMYOJOtNVokX+9SIVL2E8QoNub0JDBE4wNh97aUAPiiNvpAAMhM/eO7SWUVAx5rcDTBjf7ETtdnvXtcHaqOnK2HmNSV9K9UVy5Or3Sd+0+kxqDoWRXGE0y5pdb8+HNqwMcryszYvAv8HVcoKFgF4qd7GmzniNvZETOkrbsjMsU1+mVXTMQgh7H9z6IcGVjJozV92cDsSn91USqxOmBdwFQAkFGwPV0=,fm2_lJPETtdnvXtcHaqOnK2HmNSV9K9UVy5Or3Sd+0+kxqDoWRXGE0y5pdb8+HNqwMcryszYvAv8HVcoKFgF4qd7GmzniNvZETOkrbsjMsU1+mVXTMQQqnJP464DwxC6D4e3p9THZMO5aHR0cHM6Ly9hcGkuZmx5LmlvL2FhYS92MZgEks5oESI9zwAAAAEkCUBbF84AD2FZCpHOAA9hWQzEEL6gO8olFxMOq1uFxP1yJavEIBDKb7RuqVr/sFQniKl0S2HMM6+AQJH3940ly0mufbYx"
+
+EXPOSE 3006
+CMD ["node", "src/index.js"]

deployment-service/fly.toml

@@ -0,0 +1,19 @@
+app = 'deployment-service-test'
+primary_region = 'sea'
+
+[build]
+dockerfile = "Dockerfile"
+
+[env]
+  FLY_ORG="personal"
+  COMMON_BUCKET="snakeapi-deployment-test-bucket"
+  AWS_ACCESS_KEY_ID="tid__NSmOVaGknqitaCySppZjqVTgJSdDFnFbWcQllkC_juHwkbQZO"
+  AWS_SECRET_ACCESS_KEY="tsec_6Bz1aMbfYQftuq5WfIVEDZkHwskU4MMjVywdtxSP6uxetEBvkSC2VHI9HfTeDgHr4D6kiz"
+  AWS_ENDPOINT_URL_S3="https://fly.storage.tigris.dev"
+  AWS_REGION="auto"
+  IMAGE_REF="registry.fly.io/snake-api-template:latest"
+  FLY_API_BASE_URL = "https://api.machines.dev/v1"
+
+[http_service]
+internal_port = 3006
+force_https = true

deployment-service/package.json

@@ -0,0 +1,20 @@
+{
+  "name": "deployment-service",
+  "version": "1.0.0",
+  "description": "Manage Fly.io Notebook API servers with Tigris storage",
+  "main": "src/index.js",
+  "scripts": {
+    "start": "node src/index.js",
+    "dev": "nodemon src/index.js"
+  },
+  "dependencies": {
+    "aws-sdk": "^2.1420.0",
+    "axios": "^1.4.0",
+    "dotenv": "^16.0.3",
+    "express": "^4.18.2",
+    "tar": "^7.4.3"
+  },
+  "devDependencies": {
+    "nodemon": "^2.0.22"
+  }
+}

deployment-service/snakeapi_service/Dockerfile

@@ -0,0 +1,28 @@
+FROM python:3.11-slim
+WORKDIR /app
+
+# disable Python output buffering
+ENV PYTHONUNBUFFERED=1
+# wrapper server port
+ENV PORT=8000
+# notebook server port
+ENV NOTEBOOK_PORT=3006
+
+RUN pip install --no-cache-dir \
+    flask \
+    flask-cors \
+    awscli \
+    jupyter \
+    nbconvert \
+    nbformat \
+    requests
+
+COPY entrypoint.sh .
+COPY snakeapi_server.py .
+COPY notebooks ./notebooks
+
+RUN chmod +x entrypoint.sh
+
+EXPOSE ${PORT} ${NOTEBOOK_PORT}
+
+CMD ["./entrypoint.sh"]

deployment-service/snakeapi_service/entrypoint.sh

@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+set -eux
+
+NOTEBOOK_DIR="notebooks"
+mkdir -p "${NOTEBOOK_DIR}"
+
+# sync notebooks from S3
+aws --endpoint-url "$AWS_ENDPOINT_URL_S3" --region "$AWS_REGION" \
+  s3 sync "s3://$COMMON_BUCKET/$INSTANCE_PREFIX/notebooks/" "${NOTEBOOK_DIR}/"
+
+# pick latest notebook
+latest_ipynb=$(ls -t "${NOTEBOOK_DIR}"/*.ipynb | head -n1)
+cp "$latest_ipynb" "${NOTEBOOK_DIR}/notebook.ipynb"
+
+# convert notebook to script
+jupyter nbconvert --to script "${NOTEBOOK_DIR}/notebook.ipynb" \
+  --output notebook --output-dir "${NOTEBOOK_DIR}"
+
+# remove IPython magics only
+sed -i '/get_ipython()/d' "${NOTEBOOK_DIR}/notebook.py"
+sed -i '/^!/d'           "${NOTEBOOK_DIR}/notebook.py"
+sed -i '/^%/d'           "${NOTEBOOK_DIR}/notebook.py"
+
+# remove standalone Flask run calls
+sed -i '/app\.run(/d'    "${NOTEBOOK_DIR}/notebook.py"
+
+# export path and launch server
+export NOTEBOOK_PATH="$(pwd)/${NOTEBOOK_DIR}/notebook.py"
+exec python snakeapi_server.py

deployment-service/snakeapi_service/snakeapi_server.py

@@ -0,0 +1,55 @@
+import os
+import logging
+import importlib.util
+from flask import Flask, request, jsonify, Response
+from flask_cors import CORS
+
+logging.basicConfig(level=logging.INFO)
+
+NOTEBOOK_PATH = os.environ.get("NOTEBOOK_PATH")
+if not NOTEBOOK_PATH or not os.path.isfile(NOTEBOOK_PATH):
+    raise RuntimeError(f"NOTEBOOK_PATH not set or file not found: {NOTEBOOK_PATH}")
+
+# Determine .ipynb path
+notebook_ipynb_path = NOTEBOOK_PATH[:-3] + ".ipynb"
+
+spec = importlib.util.spec_from_file_location("notebook_module", NOTEBOOK_PATH)
+notebook_module = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(notebook_module)
+
+app = Flask(__name__)
+CORS(app)
+
+@app.route("/health", methods=["GET"])
+def health():
+    return jsonify({"status": "ok"}), 200
+
+@app.route("/", methods=["GET"])
+def info():
+    return jsonify(notebook_module.info())
+
+@app.route("/start", methods=["POST"])
+def start():
+    data = request.get_json()
+    return jsonify(notebook_module.start(data))
+
+@app.route("/move", methods=["POST"])
+def move():
+    data = request.get_json()
+    return jsonify(notebook_module.move(data))
+
+@app.route("/end", methods=["POST"])
+def end():
+    data = request.get_json()
+    return jsonify(notebook_module.end(data))
+
+@app.route("/notebook", methods=["GET"])
+def get_notebook():
+    if os.path.isfile(notebook_ipynb_path):
+        with open(notebook_ipynb_path, "r", encoding="utf-8") as f:
+            return Response(f.read(), mimetype="application/json")
+    return jsonify({"error": "notebook not found"}), 404
+
+if __name__ == "__main__":
+    port = int(os.environ.get("PORT", 8000))
+    app.run(host="0.0.0.0", port=port)

deployment-service/src/index.js

@@ -0,0 +1,231 @@
+const express = require('express');
+const fs = require('fs');
+const path = require('path');
+const AWS = require('aws-sdk');
+const axios = require('axios');
+
+const {
+  FLY_ORG,
+  COMMON_BUCKET,
+  AWS_ACCESS_KEY_ID,
+  AWS_SECRET_ACCESS_KEY,
+  AWS_ENDPOINT_URL_S3,
+  AWS_REGION,
+  FLY_ACCESS_TOKEN,
+  IMAGE_REF
+} = process.env;
+
+// Log environment variables for debugging
+console.log('--- ENV START ---');
+console.log('FLY_ORG:              ', FLY_ORG);
+console.log('COMMON_BUCKET:        ', COMMON_BUCKET);
+console.log('AWS_ACCESS_KEY_ID:    ', AWS_ACCESS_KEY_ID ? '(found)' : '(NOT SET)');
+console.log('AWS_SECRET_ACCESS_KEY:', AWS_SECRET_ACCESS_KEY ? '(found)' : '(NOT SET)');
+console.log('AWS_ENDPOINT_URL_S3:  ', AWS_ENDPOINT_URL_S3);
+console.log('AWS_REGION:           ', AWS_REGION);
+console.log('FLY_ACCESS_TOKEN:     ', FLY_ACCESS_TOKEN ? '(found)' : '(NOT SET)');
+console.log('IMAGE_REF:            ', IMAGE_REF);
+console.log('--- ENV END   ---');
+
+// Initialize S3 client
+const s3 = new AWS.S3({
+  endpoint: AWS_ENDPOINT_URL_S3,
+  region: AWS_REGION,
+  credentials: new AWS.Credentials(AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY),
+  s3ForcePathStyle: true
+});
+
+// Create Fly Machines API client
+function createFlyClient() {
+  return axios.create({
+    baseURL: 'https://api.machines.dev/v1',
+    headers: {
+      Authorization: `Bearer ${FLY_ACCESS_TOKEN}`,
+      'Content-Type': 'application/json'
+    }
+  });
+}
+
+// Create Fly GraphQL client (for IP allocation)
+const gqlClient = axios.create({
+  baseURL: 'https://api.fly.io/graphql',
+  headers: {
+    Authorization: `Bearer ${FLY_ACCESS_TOKEN}`,
+    'Content-Type': 'application/json'
+  }
+});
+
+// Define GraphQL mutation for IP allocation
+const ALLOCATE_IP_MUTATION = `
+mutation AllocateIp($input: AllocateIPAddressInput!) {
+  allocateIpAddress(input: $input) {
+    ipAddress {
+      address
+      type
+    }
+  }
+}
+`;
+
+const app = express();
+app.use(express.json({ limit: '10mb' }));
+
+app.post('/deploy', async (req, res) => {
+  console.log('Received /deploy:', req.body);
+  const { appName, region, notebookName } = req.body;
+  if (!appName || !notebookName) {
+    return res.status(400).json({ error: 'appName and notebookName required' });
+  }
+
+  const notebookPath = path.join(__dirname, '../snakeapi_service/notebooks', notebookName);
+  console.log('Resolved notebookPath:', notebookPath);
+  if (!fs.existsSync(notebookPath)) {
+    console.error('Notebook not found at:', notebookPath);
+    return res.status(500).json({ error: `Notebook not found: ${notebookPath}` });
+  }
+
+  try {
+    const fly = createFlyClient();
+
+    console.log('Creating Fly app:', appName);
+    await fly.post('/apps', {
+      app_name: appName,
+      org_slug: FLY_ORG,
+      primary_region: region
+    });
+
+    console.log('Uploading notebook to S3');
+    const data = fs.readFileSync(notebookPath);
+    const key = `${appName}/notebooks/${Date.now()}-notebook.ipynb`;
+    console.log('S3 key:', key);
+    await s3.putObject({
+      Bucket: COMMON_BUCKET,
+      Key: key,
+      Body: data,
+      ContentType: 'application/json'
+    }).promise();
+
+    console.log('Creating machine');
+    const machineConfig = {
+      name: `${appName}-machine`,
+      region: 'sea',
+      count: 1,
+      vm_size: 'shared-cpu-1x',
+      autostart: true,
+      config: {
+        image: IMAGE_REF,
+        env: {
+          INSTANCE_PREFIX: appName,
+          NOTEBOOK_KEY: key,
+          COMMON_BUCKET: COMMON_BUCKET,
+          AWS_ACCESS_KEY_ID,
+          AWS_SECRET_ACCESS_KEY,
+          AWS_ENDPOINT_URL_S3,
+          AWS_REGION
+        },
+        http_service: {
+          internal_port: 8000,
+          force_https: true,
+          auto_stop_machines: 'stop',
+          auto_start_machines: true,
+          min_machines_running: 0,
+          processes: ['app']
+        },
+        services: [
+          {
+            protocol: 'tcp',
+            internal_port: 8000,
+            ports: [
+              { port: 443, handlers: ['tls', 'http'] },
+              { port: 80,  handlers: ['http'] }
+            ]
+          }
+        ],
+        guest: {
+          memory_mb: 512,
+          cpu_kind: 'shared',
+          cpus: 1
+        }
+      }
+    };
+    console.log('Machine config:', JSON.stringify(machineConfig, null, 2));
+    await fly.post(`/apps/${appName}/machines`, machineConfig);
+
+    console.log('Allocating IPv4 via GraphQL API');
+    const v4resp = await gqlClient.post('', {
+      query: ALLOCATE_IP_MUTATION,
+      variables: { input: { appId: appName, type: 'v4' } }
+    });
+    const ipv4 = v4resp.data.data.allocateIpAddress.ipAddress.address;
+    console.log('Allocated IPv4:', ipv4);
+
+    console.log('Allocating IPv6 via GraphQL API');
+    const v6resp = await gqlClient.post('', {
+      query: ALLOCATE_IP_MUTATION,
+      variables: { input: { appId: appName, type: 'v6' } }
+    });
+    const ipv6 = v6resp.data.data.allocateIpAddress.ipAddress.address;
+    console.log('Allocated IPv6:', ipv6);
+
+    return res.json({
+      status: 'created',
+      app: appName,
+      ipv4,
+      ipv6,
+      url_v4: `http://${ipv4}`,
+      url_v6: `http://[${ipv6}]`
+    });
+  } catch (err) {
+    console.error('Deployment error:', err.response?.data || err.stack || err.message);
+    return res.status(500).json({ error: err.response?.data || err.message });
+  }
+});
+
+// Upload notebook to S3 for an existing app
+app.post('/:appName/upload', async (req, res) => {
+  const { appName } = req.params;
+  const { notebookName, fileContentBase64 } = req.body;
+
+  if (!notebookName || !fileContentBase64) {
+    return res.status(400).json({ error: 'notebookName and fileContentBase64 are required' });
+  }
+
+  try {
+    const buffer = Buffer.from(fileContentBase64, 'base64');
+    const key = `${appName}/notebooks/${notebookName}`;
+
+    console.log(`Uploading notebook to: s3://${COMMON_BUCKET}/${key}`);
+    await s3.putObject({
+      Bucket: COMMON_BUCKET,
+      Key: key,
+      Body: buffer,
+      ContentType: 'application/json'
+    }).promise();
+
+    return res.json({ status: 'uploaded', app: appName, key });
+  } catch (err) {
+    console.error('Notebook upload error:', err);
+    return res.status(500).json({ error: err.message });
+  }
+});
+
+// Delete a Fly app
+app.post('/:appName/delete', async (req, res) => {
+  const { appName } = req.params;
+
+  try {
+    const fly = createFlyClient();
+    console.log('Destroying Fly app:', appName);
+    await fly.delete(`/apps/${appName}`);
+
+    return res.json({ status: 'deleted', app: appName });
+  } catch (err) {
+    console.error('App deletion error:', err.response?.data || err.message);
+    return res.status(500).json({ error: err.response?.data || err.message });
+  }
+});
+
+const LISTEN_PORT = process.env.PORT || 3006;
+app.listen(LISTEN_PORT, '0.0.0.0', () => {
+  console.log(`Deployment service listening on port ${LISTEN_PORT}`);
+});

package-lock.json

@@ -0,0 +1,6 @@
+{
+  "name": "bytecamp-services",
+  "lockfileVersion": 3,
+  "requires": true,
+  "packages": {}
+}