From 61650658ebcbe4967832eaf68cf3717c0d59a67d Mon Sep 17 00:00:00 2001 From: yoshi Date: Tue, 29 Apr 2025 13:28:25 -0700 Subject: [PATCH] modified index.js --- deployment-service/package-lock.json | 86 ++++++++++++++++++- deployment-service/package.json | 5 +- .../snakeapi_service/entrypoint.sh | 20 ++--- deployment-service/src/index.js | 60 +++---------- 4 files changed, 106 insertions(+), 65 deletions(-) diff --git a/deployment-service/package-lock.json b/deployment-service/package-lock.json index 5b6e5e0..f775c84 100644 --- a/deployment-service/package-lock.json +++ b/deployment-service/package-lock.json @@ -11,12 +11,25 @@ "aws-sdk": "^2.1420.0", "axios": "^1.4.0", "dotenv": "^16.0.3", - "express": "^4.18.2" + "express": "^4.18.2", + "tar": "^7.4.3" }, "devDependencies": { "nodemon": "^2.0.22" } }, + "node_modules/@isaacs/fs-minipass": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/@isaacs/fs-minipass/-/fs-minipass-4.0.1.tgz", + "integrity": "sha512-wgm9Ehl2jpeqP3zw/7mo3kRHFp5MEDhqAdwy1fTGkHAwnkGOVsgpvQhL8B5n1qlb01jV3n/bI0ZfZp5lWA1k4w==", + "license": "ISC", + "dependencies": { + "minipass": "^7.0.4" + }, + "engines": { + "node": ">=18.0.0" + } + }, "node_modules/accepts": { "version": "1.3.8", "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.8.tgz", @@ -284,6 +297,15 @@ "fsevents": "~2.3.2" } }, + "node_modules/chownr": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/chownr/-/chownr-3.0.0.tgz", + "integrity": "sha512-+IxzY9BZOQd/XuYPRmrvEVjF/nqj5kgT4kEq7VofrDoM1MxoRjEWkrCC3EtLi59TVawxTAn+orJwFQcrqEN1+g==", + "license": "BlueOak-1.0.0", + "engines": { + "node": ">=18" + } + }, "node_modules/combined-stream": { "version": "1.0.8", "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", @@ -1073,6 +1095,42 @@ "node": "*" } }, + "node_modules/minipass": { + "version": "7.1.2", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.2.tgz", + "integrity": "sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw==", + "license": "ISC", + "engines": { + "node": ">=16 || 14 >=14.17" + } + }, + "node_modules/minizlib": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-3.0.2.tgz", + "integrity": "sha512-oG62iEk+CYt5Xj2YqI5Xi9xWUeZhDI8jjQmC5oThVH5JGCTgIjr7ciJDzC7MBzYd//WvR1OTmP5Q38Q8ShQtVA==", + "license": "MIT", + "dependencies": { + "minipass": "^7.1.2" + }, + "engines": { + "node": ">= 18" + } + }, + "node_modules/mkdirp": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-3.0.1.tgz", + "integrity": "sha512-+NsyUUAZDmo6YVHzL/stxSu3t9YS1iljliy3BSDrXJ/dkn1KYdmtZODGGjLcc9XLgVVpH4KshHB8XmZgMhaBXg==", + "license": "MIT", + "bin": { + "mkdirp": "dist/cjs/src/bin.js" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/ms": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", @@ -1551,6 +1609,23 @@ "node": ">=4" } }, + "node_modules/tar": { + "version": "7.4.3", + "resolved": "https://registry.npmjs.org/tar/-/tar-7.4.3.tgz", + "integrity": "sha512-5S7Va8hKfV7W5U6g3aYxXmlPoZVAwUMy9AOKyF2fVuZa2UD3qZjg578OrLRt8PcNN1PleVaL/5/yYATNL0ICUw==", + "license": "ISC", + "dependencies": { + "@isaacs/fs-minipass": "^4.0.0", + "chownr": "^3.0.0", + "minipass": "^7.1.2", + "minizlib": "^3.0.1", + "mkdirp": "^3.0.1", + "yallist": "^5.0.0" + }, + "engines": { + "node": ">=18" + } + }, "node_modules/to-regex-range": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", @@ -1704,6 +1779,15 @@ "engines": { "node": ">=4.0" } + }, + "node_modules/yallist": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-5.0.0.tgz", + "integrity": "sha512-YgvUTfwqyc7UXVMrB+SImsVYSmTS8X/tSrtdNZMImM+n7+QTriRXyXim0mBrTXNeqzVF0KWGgHPeiyViFFrNDw==", + "license": "BlueOak-1.0.0", + "engines": { + "node": ">=18" + } } } } diff --git a/deployment-service/package.json b/deployment-service/package.json index 604019d..440cd07 100644 --- a/deployment-service/package.json +++ b/deployment-service/package.json @@ -8,10 +8,11 @@ "dev": "nodemon src/index.js" }, "dependencies": { - "express": "^4.18.2", + "aws-sdk": "^2.1420.0", "axios": "^1.4.0", "dotenv": "^16.0.3", - "aws-sdk": "^2.1420.0" + "express": "^4.18.2", + "tar": "^7.4.3" }, "devDependencies": { "nodemon": "^2.0.22" diff --git a/deployment-service/snakeapi_service/entrypoint.sh b/deployment-service/snakeapi_service/entrypoint.sh index 0b5fea6..38688b5 100644 --- a/deployment-service/snakeapi_service/entrypoint.sh +++ b/deployment-service/snakeapi_service/entrypoint.sh @@ -1,21 +1,17 @@ #!/usr/bin/env bash -last_mod=0 +NOTEBOOK_DIR="notebooks" +mkdir -p ${NOTEBOOK_DIR} while true; do aws --endpoint-url "$AWS_ENDPOINT_URL_S3" --region "$AWS_REGION" \ - s3 sync "s3://$BUCKET_NAME/$INSTANCE_PREFIX" . --exclude "*" --include "notebook.ipynb" + s3 sync "s3://$BUCKET_NAME/$INSTANCE_PREFIX/notebooks/" "${NOTEBOOK_DIR}/" - if [ -f "notebook.ipynb" ]; then - new_mod=$(stat -c %Y notebook.ipynb) - else - new_mod=0 + latest_notebook=$(ls -t ${NOTEBOOK_DIR}/*.ipynb | head -1) + + if [ -n "$latest_notebook" ]; then + jupyter nbconvert --to notebook --execute --inplace --ExecutePreprocessor.timeout=0 "$latest_notebook" fi - if [ "$new_mod" -ne "$last_mod" ]; then - last_mod=$new_mod - jupyter nbconvert --to notebook --execute --inplace --ExecutePreprocessor.timeout=0 notebook.ipynb - fi - - sleep 1 + sleep 5 done diff --git a/deployment-service/src/index.js b/deployment-service/src/index.js index 1402747..ffeca4c 100644 --- a/deployment-service/src/index.js +++ b/deployment-service/src/index.js @@ -24,7 +24,7 @@ const s3 = new AWS.S3({ function createFlyClient() { return axios.create({ - baseURL: 'https://api.fly.io', + baseURL: 'https://api.machines.dev/v1', headers: { Authorization: `Bearer ${FLY_ACCESS_TOKEN}`, 'Content-Type': 'application/json' @@ -45,7 +45,7 @@ app.post('/deploy', async (req, res) => { const fly = createFlyClient(); await fly.post('/apps', { - name: appName, + app_name: appName, org_slug: FLY_ORG, primary_region: region }); @@ -62,10 +62,17 @@ app.post('/deploy', async (req, res) => { }); const notebookFile = path.join(__dirname, '../snakeapi_service/notebooks', notebookName); + + if (!fs.existsSync(notebookFile)) { + throw new Error(`Notebook file ${notebookName} not found.`); + } + const notebookData = fs.readFileSync(notebookFile); + const timestamp = Date.now(); + await s3.putObject({ Bucket: COMMON_BUCKET, - Key: `${appName}/notebook.ipynb`, + Key: `${appName}/notebooks/${timestamp}-notebook.ipynb`, Body: notebookData, ContentType: 'application/json' }).promise(); @@ -95,52 +102,5 @@ app.post('/deploy', async (req, res) => { } }); -app.post('/upload/:appName', async (req, res) => { - const { appName } = req.params; - const notebookContent = req.body; - - if (!notebookContent) { - return res.status(400).json({ error: 'Notebook content required.' }); - } - - try { - const notebookBuffer = Buffer.from(JSON.stringify(notebookContent)); - await s3.putObject({ - Bucket: COMMON_BUCKET, - Key: `${appName}/notebook.ipynb`, - Body: notebookBuffer, - ContentType: 'application/json' - }).promise(); - - const fly = createFlyClient(); - - const tempNotebookPath = path.join(__dirname, '../snakeapi_service/notebooks/notebook.ipynb'); - fs.writeFileSync(tempNotebookPath, notebookBuffer); - - const tarFilePath = `/tmp/${appName}-redeploy.tar.gz`; - await tar.c( - { - gzip: true, - file: tarFilePath, - cwd: path.join(__dirname, '../snakeapi_service') - }, - ['.'] - ); - - const tarData = fs.readFileSync(tarFilePath); - await fly.post(`/apps/${appName}/deploys`, tarData, { - headers: { 'Content-Type': 'application/gzip' } - }); - - res.json({ - status: 'updated', - app: appName, - message: 'Notebook updated and application redeployed.' - }); - } catch (error) { - res.status(500).json({ error: error.response?.data || error.message }); - } -}); - const port = process.env.PORT || 3006; app.listen(port, '0.0.0.0', () => console.log(`Listening on port ${port}`));