modified index.js

deployment-service/package-lock.json

@@ -11,12 +11,25 @@
         "aws-sdk": "^2.1420.0",
         "axios": "^1.4.0",
         "dotenv": "^16.0.3",
-        "express": "^4.18.2"
+        "express": "^4.18.2",
+        "tar": "^7.4.3"
       },
       "devDependencies": {
         "nodemon": "^2.0.22"
       }
     },
+    "node_modules/@isaacs/fs-minipass": {
+      "version": "4.0.1",
+      "resolved": "https://registry.npmjs.org/@isaacs/fs-minipass/-/fs-minipass-4.0.1.tgz",
+      "integrity": "sha512-wgm9Ehl2jpeqP3zw/7mo3kRHFp5MEDhqAdwy1fTGkHAwnkGOVsgpvQhL8B5n1qlb01jV3n/bI0ZfZp5lWA1k4w==",
+      "license": "ISC",
+      "dependencies": {
+        "minipass": "^7.0.4"
+      },
+      "engines": {
+        "node": ">=18.0.0"
+      }
+    },
     "node_modules/accepts": {
       "version": "1.3.8",
       "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.8.tgz",
@@ -284,6 +297,15 @@
         "fsevents": "~2.3.2"
       }
     },
+    "node_modules/chownr": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/chownr/-/chownr-3.0.0.tgz",
+      "integrity": "sha512-+IxzY9BZOQd/XuYPRmrvEVjF/nqj5kgT4kEq7VofrDoM1MxoRjEWkrCC3EtLi59TVawxTAn+orJwFQcrqEN1+g==",
+      "license": "BlueOak-1.0.0",
+      "engines": {
+        "node": ">=18"
+      }
+    },
     "node_modules/combined-stream": {
       "version": "1.0.8",
       "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz",
@@ -1073,6 +1095,42 @@
         "node": "*"
       }
     },
+    "node_modules/minipass": {
+      "version": "7.1.2",
+      "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.2.tgz",
+      "integrity": "sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw==",
+      "license": "ISC",
+      "engines": {
+        "node": ">=16 || 14 >=14.17"
+      }
+    },
+    "node_modules/minizlib": {
+      "version": "3.0.2",
+      "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-3.0.2.tgz",
+      "integrity": "sha512-oG62iEk+CYt5Xj2YqI5Xi9xWUeZhDI8jjQmC5oThVH5JGCTgIjr7ciJDzC7MBzYd//WvR1OTmP5Q38Q8ShQtVA==",
+      "license": "MIT",
+      "dependencies": {
+        "minipass": "^7.1.2"
+      },
+      "engines": {
+        "node": ">= 18"
+      }
+    },
+    "node_modules/mkdirp": {
+      "version": "3.0.1",
+      "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-3.0.1.tgz",
+      "integrity": "sha512-+NsyUUAZDmo6YVHzL/stxSu3t9YS1iljliy3BSDrXJ/dkn1KYdmtZODGGjLcc9XLgVVpH4KshHB8XmZgMhaBXg==",
+      "license": "MIT",
+      "bin": {
+        "mkdirp": "dist/cjs/src/bin.js"
+      },
+      "engines": {
+        "node": ">=10"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/isaacs"
+      }
+    },
     "node_modules/ms": {
       "version": "2.0.0",
       "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
@@ -1551,6 +1609,23 @@
         "node": ">=4"
       }
     },
+    "node_modules/tar": {
+      "version": "7.4.3",
+      "resolved": "https://registry.npmjs.org/tar/-/tar-7.4.3.tgz",
+      "integrity": "sha512-5S7Va8hKfV7W5U6g3aYxXmlPoZVAwUMy9AOKyF2fVuZa2UD3qZjg578OrLRt8PcNN1PleVaL/5/yYATNL0ICUw==",
+      "license": "ISC",
+      "dependencies": {
+        "@isaacs/fs-minipass": "^4.0.0",
+        "chownr": "^3.0.0",
+        "minipass": "^7.1.2",
+        "minizlib": "^3.0.1",
+        "mkdirp": "^3.0.1",
+        "yallist": "^5.0.0"
+      },
+      "engines": {
+        "node": ">=18"
+      }
+    },
     "node_modules/to-regex-range": {
       "version": "5.0.1",
       "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz",
@@ -1704,6 +1779,15 @@
       "engines": {
         "node": ">=4.0"
       }
+    },
+    "node_modules/yallist": {
+      "version": "5.0.0",
+      "resolved": "https://registry.npmjs.org/yallist/-/yallist-5.0.0.tgz",
+      "integrity": "sha512-YgvUTfwqyc7UXVMrB+SImsVYSmTS8X/tSrtdNZMImM+n7+QTriRXyXim0mBrTXNeqzVF0KWGgHPeiyViFFrNDw==",
+      "license": "BlueOak-1.0.0",
+      "engines": {
+        "node": ">=18"
+      }
     }
   }
 }

deployment-service/package.json

@@ -8,10 +8,11 @@
     "dev": "nodemon src/index.js"
   },
   "dependencies": {
-    "express": "^4.18.2",
+    "aws-sdk": "^2.1420.0",
     "axios": "^1.4.0",
     "dotenv": "^16.0.3",
-    "aws-sdk": "^2.1420.0"
+    "express": "^4.18.2",
+    "tar": "^7.4.3"
   },
   "devDependencies": {
     "nodemon": "^2.0.22"

deployment-service/snakeapi_service/entrypoint.sh

@@ -1,21 +1,17 @@
 #!/usr/bin/env bash
 
-last_mod=0
+NOTEBOOK_DIR="notebooks"
+mkdir -p ${NOTEBOOK_DIR}
 
 while true; do
   aws --endpoint-url "$AWS_ENDPOINT_URL_S3" --region "$AWS_REGION" \
-    s3 sync "s3://$BUCKET_NAME/$INSTANCE_PREFIX" . --exclude "*" --include "notebook.ipynb"
+    s3 sync "s3://$BUCKET_NAME/$INSTANCE_PREFIX/notebooks/" "${NOTEBOOK_DIR}/"
 
-  if [ -f "notebook.ipynb" ]; then
-    new_mod=$(stat -c %Y notebook.ipynb)
-  else
-    new_mod=0
+  latest_notebook=$(ls -t ${NOTEBOOK_DIR}/*.ipynb | head -1)
+
+  if [ -n "$latest_notebook" ]; then
+    jupyter nbconvert --to notebook --execute --inplace --ExecutePreprocessor.timeout=0 "$latest_notebook"
   fi
 
-  if [ "$new_mod" -ne "$last_mod" ]; then
-    last_mod=$new_mod
-    jupyter nbconvert --to notebook --execute --inplace --ExecutePreprocessor.timeout=0 notebook.ipynb
-  fi
-
-  sleep 1
+  sleep 5
 done

deployment-service/src/index.js

@@ -24,7 +24,7 @@ const s3 = new AWS.S3({
 
 function createFlyClient() {
   return axios.create({
-    baseURL: 'https://api.fly.io',
+    baseURL: 'https://api.machines.dev/v1',
     headers: {
       Authorization: `Bearer ${FLY_ACCESS_TOKEN}`,
       'Content-Type': 'application/json'
@@ -45,7 +45,7 @@ app.post('/deploy', async (req, res) => {
     const fly = createFlyClient();
 
     await fly.post('/apps', {
-      name: appName,
+      app_name: appName,
       org_slug: FLY_ORG,
       primary_region: region
     });
@@ -62,10 +62,17 @@ app.post('/deploy', async (req, res) => {
     });
 
     const notebookFile = path.join(__dirname, '../snakeapi_service/notebooks', notebookName);
+
+    if (!fs.existsSync(notebookFile)) {
+      throw new Error(`Notebook file ${notebookName} not found.`);
+    }
+
     const notebookData = fs.readFileSync(notebookFile);
+    const timestamp = Date.now();
+
     await s3.putObject({
       Bucket: COMMON_BUCKET,
-      Key: `${appName}/notebook.ipynb`,
+      Key: `${appName}/notebooks/${timestamp}-notebook.ipynb`,
       Body: notebookData,
       ContentType: 'application/json'
     }).promise();
@@ -95,52 +102,5 @@ app.post('/deploy', async (req, res) => {
   }
 });
 
-app.post('/upload/:appName', async (req, res) => {
-  const { appName } = req.params;
-  const notebookContent = req.body;
-
-  if (!notebookContent) {
-    return res.status(400).json({ error: 'Notebook content required.' });
-  }
-
-  try {
-    const notebookBuffer = Buffer.from(JSON.stringify(notebookContent));
-    await s3.putObject({
-      Bucket: COMMON_BUCKET,
-      Key: `${appName}/notebook.ipynb`,
-      Body: notebookBuffer,
-      ContentType: 'application/json'
-    }).promise();
-
-    const fly = createFlyClient();
-
-    const tempNotebookPath = path.join(__dirname, '../snakeapi_service/notebooks/notebook.ipynb');
-    fs.writeFileSync(tempNotebookPath, notebookBuffer);
-
-    const tarFilePath = `/tmp/${appName}-redeploy.tar.gz`;
-    await tar.c(
-      {
-        gzip: true,
-        file: tarFilePath,
-        cwd: path.join(__dirname, '../snakeapi_service')
-      },
-      ['.']
-    );
-
-    const tarData = fs.readFileSync(tarFilePath);
-    await fly.post(`/apps/${appName}/deploys`, tarData, {
-      headers: { 'Content-Type': 'application/gzip' }
-    });
-
-    res.json({
-      status: 'updated',
-      app: appName,
-      message: 'Notebook updated and application redeployed.'
-    });
-  } catch (error) {
-    res.status(500).json({ error: error.response?.data || error.message });
-  }
-});
-
 const port = process.env.PORT || 3006;
 app.listen(port, '0.0.0.0', () => console.log(`Listening on port ${port}`));