Merge pull request #131 from mcataford/refactor/parsing-in-func
refactor: move XML parsing to backend
This commit is contained in:
GitHub
commit
f10751e077
9 changed files with 137 additions and 67 deletions
|
|
@ -3,6 +3,9 @@
|
|||
"organizeImports": {
|
||||
"enabled": false
|
||||
},
|
||||
"files": {
|
||||
"ignore": ["node_modules/**/*", ".netlify/**/*", "dist/**/*"]
|
||||
},
|
||||
"linter": {
|
||||
"enabled": true,
|
||||
"rules": {
|
||||
|
|
|
|||
|
|
@ -1,26 +0,0 @@
|
|||
const https = require('https')
|
||||
|
||||
async function httpGet(url) {
|
||||
return new Promise((resolve) => {
|
||||
https.get(url, (response) => {
|
||||
const chunks = []
|
||||
response.on('data', (d) => chunks.push(d))
|
||||
response.on('end', () => resolve(chunks.join('')))
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
const handler = async (event) => {
|
||||
try {
|
||||
const url = event.queryStringParameters.url
|
||||
const proxiedResponse = await httpGet(url)
|
||||
return {
|
||||
statusCode: 200,
|
||||
body: String(proxiedResponse),
|
||||
}
|
||||
} catch (error) {
|
||||
return { statusCode: 500, body: error.toString() }
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { handler }
|
||||
49
netlify/functions/rss-proxy/rss-proxy.ts
Executable file
49
netlify/functions/rss-proxy/rss-proxy.ts
Executable file
|
|
@ -0,0 +1,49 @@
|
|||
/*
|
||||
* Fetches and parses RSS feeds.
|
||||
*
|
||||
* This handles the fetching, XML parsing and formatting of
|
||||
* RSS feed data so that the frontent clients do not have to.
|
||||
*
|
||||
* This is operating on a "by-feed" basis such that each
|
||||
* run only processes one feed, and the clients are expected
|
||||
* to make multiple requests if they have a list of feeds to
|
||||
* follow.
|
||||
*/
|
||||
|
||||
import axios from "axios";
|
||||
import { parseFeed } from "htmlparser2";
|
||||
|
||||
function processFeedXML(feed) {
|
||||
return {
|
||||
title: feed.title,
|
||||
lastPull: String(Date.now()),
|
||||
items: feed.items.reduce((items, feedItem) => {
|
||||
items.push({
|
||||
title: feedItem.title,
|
||||
url: feedItem.link,
|
||||
published: new Date(feedItem.pubDate),
|
||||
});
|
||||
|
||||
return items;
|
||||
}, []),
|
||||
};
|
||||
}
|
||||
|
||||
const handler = async (event) => {
|
||||
try {
|
||||
const url = event.queryStringParameters.url;
|
||||
const responseData = await axios.get(url);
|
||||
const newFeedData = parseFeed(responseData.data);
|
||||
const newFeed = processFeedXML(newFeedData);
|
||||
const mergedFeeds = newFeed;
|
||||
|
||||
return {
|
||||
statusCode: 200,
|
||||
body: JSON.stringify(mergedFeeds),
|
||||
};
|
||||
} catch (error) {
|
||||
return { statusCode: 500, body: error.toString() };
|
||||
}
|
||||
};
|
||||
|
||||
module.exports = { handler };
|
||||
|
|
@ -5,8 +5,8 @@
|
|||
"packageManager": "yarn@4.1.0",
|
||||
"license": "GPL-3.0",
|
||||
"scripts": {
|
||||
"lint": "yarn biome format src/**/*.ts src/**/*.tsx && yarn biome check src/**/*.ts src/**/*.tsx",
|
||||
"lint:fix": "yarn biome format src/**/*.ts src/**/*.tsx --write && yarn biome check src/**/*.ts src/**/*.tsx --apply",
|
||||
"lint": "yarn biome format --config-path=. . && yarn biome check --config-path=. .",
|
||||
"lint:fix": "yarn biome format --config-path=. . --write && yarn biome check --config-path=. . --apply",
|
||||
"typecheck": "tsc --noEmit",
|
||||
"start": "netlify dev",
|
||||
"start:app": "vite ./src --config ./vite.config.js --port 8080",
|
||||
|
|
@ -19,12 +19,14 @@
|
|||
"@mui/icons-material": "^5.15.10",
|
||||
"@mui/material": "^5.15.10",
|
||||
"@tanstack/react-query": "^5.20.5",
|
||||
"axios": "^1.6.7",
|
||||
"htmlparser2": "^9.1.0",
|
||||
"react": "^18.2.0",
|
||||
"react-dom": "^18.2.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@biomejs/biome": "^1.5.3",
|
||||
"@netlify/functions": "^2.6.0",
|
||||
"@types/react": "^18",
|
||||
"@types/react-dom": "^18",
|
||||
"@vitejs/plugin-basic-ssl": "^1.1.0",
|
||||
|
|
|
|||
|
|
@ -1,7 +1,5 @@
|
|||
{
|
||||
"$schema": "https://docs.renovatebot.com/renovate-schema.json",
|
||||
"schedule": ["on friday"],
|
||||
"extends": [
|
||||
"config:base"
|
||||
]
|
||||
"$schema": "https://docs.renovatebot.com/renovate-schema.json",
|
||||
"schedule": ["on friday"],
|
||||
"extends": ["config:base"]
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,4 +1,3 @@
|
|||
import { parseFeed } from "htmlparser2";
|
||||
import { useQueries } from "@tanstack/react-query";
|
||||
|
||||
import { Feed } from "../types";
|
||||
|
|
@ -26,22 +25,6 @@ function mergeFeeds(first, second) {
|
|||
};
|
||||
}
|
||||
|
||||
function processFeedXML(feed): Feed {
|
||||
return {
|
||||
title: feed.title,
|
||||
lastPull: String(Date.now()),
|
||||
items: feed.items.reduce((items, feedItem) => {
|
||||
items.push({
|
||||
title: feedItem.title,
|
||||
url: feedItem.link,
|
||||
published: new Date(feedItem.pubDate),
|
||||
});
|
||||
|
||||
return items;
|
||||
}, []),
|
||||
};
|
||||
}
|
||||
|
||||
async function fetchFeed(
|
||||
url: string,
|
||||
persistedData: Feed | null,
|
||||
|
|
@ -51,12 +34,10 @@ async function fetchFeed(
|
|||
const responseData = await response.text();
|
||||
|
||||
try {
|
||||
const newFeedData = parseFeed(responseData);
|
||||
const newFeed = processFeedXML(newFeedData);
|
||||
const fetched = JSON.parse(responseData);
|
||||
const mergedFeeds = persistedData
|
||||
? mergeFeeds(persistedData, newFeed)
|
||||
: newFeed;
|
||||
|
||||
? mergeFeeds(persistedData, fetched)
|
||||
: fetched;
|
||||
return mergedFeeds;
|
||||
} catch (e) {
|
||||
if (isDev()) {
|
||||
|
|
|
|||
|
|
@ -8,5 +8,5 @@
|
|||
"skipLibCheck": true,
|
||||
"rootDir": "."
|
||||
},
|
||||
"include": ["src/**/*"]
|
||||
"include": ["src/**/*", "netlify/**/*"]
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
import legacy from "@vitejs/plugin-legacy"
|
||||
import basicSSL from "@vitejs/plugin-basic-ssl"
|
||||
import path from "node:path"
|
||||
import { defineConfig } from "vite"
|
||||
import legacy from "@vitejs/plugin-legacy";
|
||||
import basicSSL from "@vitejs/plugin-basic-ssl";
|
||||
import path from "node:path";
|
||||
import { defineConfig } from "vite";
|
||||
|
||||
export default defineConfig({
|
||||
plugins: [legacy(), basicSSL()],
|
||||
|
|
@ -30,4 +30,4 @@ export default defineConfig({
|
|||
},
|
||||
],
|
||||
},
|
||||
})
|
||||
});
|
||||
|
|
|
|||
75
yarn.lock
75
yarn.lock
|
|
@ -2944,6 +2944,15 @@ __metadata:
|
|||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"@netlify/functions@npm:^2.6.0":
|
||||
version: 2.6.0
|
||||
resolution: "@netlify/functions@npm:2.6.0"
|
||||
dependencies:
|
||||
"@netlify/serverless-functions-api": "npm:1.14.0"
|
||||
checksum: 10/77afb7ffb20ed4f7b732a82a4c32e59f447119d5557ce157566e4a9e8f8f5ca34666e68daf6e0d1cfbee7170756e6acaa6dad3fdb03e8165ed77c2a367271105
|
||||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"@netlify/git-utils@npm:^5.1.1":
|
||||
version: 5.1.1
|
||||
resolution: "@netlify/git-utils@npm:5.1.1"
|
||||
|
|
@ -3149,7 +3158,7 @@ __metadata:
|
|||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"@netlify/serverless-functions-api@npm:^1.14.0":
|
||||
"@netlify/serverless-functions-api@npm:1.14.0, @netlify/serverless-functions-api@npm:^1.14.0":
|
||||
version: 1.14.0
|
||||
resolution: "@netlify/serverless-functions-api@npm:1.14.0"
|
||||
dependencies:
|
||||
|
|
@ -4733,6 +4742,13 @@ __metadata:
|
|||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"asynckit@npm:^0.4.0":
|
||||
version: 0.4.0
|
||||
resolution: "asynckit@npm:0.4.0"
|
||||
checksum: 10/3ce727cbc78f69d6a4722517a58ee926c8c21083633b1d3fdf66fd688f6c127a53a592141bd4866f9b63240a86e9d8e974b13919450bd17fa33c2d22c4558ad8
|
||||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"atob@npm:^2.1.2":
|
||||
version: 2.1.2
|
||||
resolution: "atob@npm:2.1.2"
|
||||
|
|
@ -4760,6 +4776,17 @@ __metadata:
|
|||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"axios@npm:^1.6.7":
|
||||
version: 1.6.7
|
||||
resolution: "axios@npm:1.6.7"
|
||||
dependencies:
|
||||
follow-redirects: "npm:^1.15.4"
|
||||
form-data: "npm:^4.0.0"
|
||||
proxy-from-env: "npm:^1.1.0"
|
||||
checksum: 10/a1932b089ece759cd261f175d9ebf4d41c8994cf0c0767cda86055c7a19bcfdade8ae3464bf4cec4c8b142f4a657dc664fb77a41855e8376cf38b86d7a86518f
|
||||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"b4a@npm:^1.6.4":
|
||||
version: 1.6.6
|
||||
resolution: "b4a@npm:1.6.6"
|
||||
|
|
@ -5750,6 +5777,15 @@ __metadata:
|
|||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"combined-stream@npm:^1.0.8":
|
||||
version: 1.0.8
|
||||
resolution: "combined-stream@npm:1.0.8"
|
||||
dependencies:
|
||||
delayed-stream: "npm:~1.0.0"
|
||||
checksum: 10/2e969e637d05d09fa50b02d74c83a1186f6914aae89e6653b62595cc75a221464f884f55f231b8f4df7a49537fba60bdc0427acd2bf324c09a1dbb84837e36e4
|
||||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"commander@npm:10.0.1, commander@npm:^10.0.1":
|
||||
version: 10.0.1
|
||||
resolution: "commander@npm:10.0.1"
|
||||
|
|
@ -6310,6 +6346,13 @@ __metadata:
|
|||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"delayed-stream@npm:~1.0.0":
|
||||
version: 1.0.0
|
||||
resolution: "delayed-stream@npm:1.0.0"
|
||||
checksum: 10/46fe6e83e2cb1d85ba50bd52803c68be9bd953282fa7096f51fc29edd5d67ff84ff753c51966061e5ba7cb5e47ef6d36a91924eddb7f3f3483b1c560f77a0020
|
||||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"delegates@npm:^1.0.0":
|
||||
version: 1.0.0
|
||||
resolution: "delegates@npm:1.0.0"
|
||||
|
|
@ -7679,13 +7722,13 @@ __metadata:
|
|||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"follow-redirects@npm:^1.0.0":
|
||||
version: 1.14.1
|
||||
resolution: "follow-redirects@npm:1.14.1"
|
||||
"follow-redirects@npm:^1.0.0, follow-redirects@npm:^1.15.4":
|
||||
version: 1.15.5
|
||||
resolution: "follow-redirects@npm:1.15.5"
|
||||
peerDependenciesMeta:
|
||||
debug:
|
||||
optional: true
|
||||
checksum: 10/e23bc0e42901016b563fb9356525c55f583c112b93ab623ea917596acac1e27250efdaf069bdcbc7f404b8cf4689c75769b16b47a0f3697201d2398f8054c7e5
|
||||
checksum: 10/d467f13c1c6aa734599b8b369cd7a625b20081af358f6204ff515f6f4116eb440de9c4e0c49f10798eeb0df26c95dd05d5e0d9ddc5786ab1a8a8abefe92929b4
|
||||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
|
|
@ -7703,6 +7746,17 @@ __metadata:
|
|||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"form-data@npm:^4.0.0":
|
||||
version: 4.0.0
|
||||
resolution: "form-data@npm:4.0.0"
|
||||
dependencies:
|
||||
asynckit: "npm:^0.4.0"
|
||||
combined-stream: "npm:^1.0.8"
|
||||
mime-types: "npm:^2.1.12"
|
||||
checksum: 10/7264aa760a8cf09482816d8300f1b6e2423de1b02bba612a136857413fdc96d7178298ced106817655facc6b89036c6e12ae31c9eb5bdc16aabf502ae8a5d805
|
||||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"formdata-polyfill@npm:^4.0.10":
|
||||
version: 4.0.10
|
||||
resolution: "formdata-polyfill@npm:4.0.10"
|
||||
|
|
@ -10366,7 +10420,7 @@ __metadata:
|
|||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"mime-types@npm:~2.1.24, mime-types@npm:~2.1.34":
|
||||
"mime-types@npm:^2.1.12, mime-types@npm:~2.1.24, mime-types@npm:~2.1.34":
|
||||
version: 2.1.35
|
||||
resolution: "mime-types@npm:2.1.35"
|
||||
dependencies:
|
||||
|
|
@ -12036,6 +12090,13 @@ __metadata:
|
|||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"proxy-from-env@npm:^1.1.0":
|
||||
version: 1.1.0
|
||||
resolution: "proxy-from-env@npm:1.1.0"
|
||||
checksum: 10/f0bb4a87cfd18f77bc2fba23ae49c3b378fb35143af16cc478171c623eebe181678f09439707ad80081d340d1593cd54a33a0113f3ccb3f4bc9451488780ee23
|
||||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"ps-list@npm:^8.0.0":
|
||||
version: 8.1.1
|
||||
resolution: "ps-list@npm:8.1.1"
|
||||
|
|
@ -12739,11 +12800,13 @@ __metadata:
|
|||
"@emotion/styled": "npm:^11.11.0"
|
||||
"@mui/icons-material": "npm:^5.15.10"
|
||||
"@mui/material": "npm:^5.15.10"
|
||||
"@netlify/functions": "npm:^2.6.0"
|
||||
"@tanstack/react-query": "npm:^5.20.5"
|
||||
"@types/react": "npm:^18"
|
||||
"@types/react-dom": "npm:^18"
|
||||
"@vitejs/plugin-basic-ssl": "npm:^1.1.0"
|
||||
"@vitejs/plugin-legacy": "npm:^5.3.0"
|
||||
axios: "npm:^1.6.7"
|
||||
htmlparser2: "npm:^9.1.0"
|
||||
jest: "npm:29.3.1"
|
||||
netlify-cli: "npm:^17.16.2"
|
||||
|
|
|
|||
Loading…
Reference in a new issue