Skip to content

Instantly share code, notes, and snippets.

@davidwebca
Created June 13, 2024 14:38
Show Gist options
  • Save davidwebca/12464faa2df26576f7790b4a1880bf6e to your computer and use it in GitHub Desktop.
Save davidwebca/12464faa2df26576f7790b4a1880bf6e to your computer and use it in GitHub Desktop.
Simple ajax cache warmer based on sitemap.xml
/**
* This cache warmer uses everything located in the sitemap.xml
* of the current opened website. Just copy and paste all of this
* in the browser console while the target website is opened.
*/​
async function crawl() {
let urls = [];
async function getUrls() {
await fetch(`${window.location.origin}/sitemap.xml`).then(response => response.text())
.then(str => new window.DOMParser().parseFromString(str.trimStart(), "text/xml"))
.then(data => {
let otherSitemaps = Array.from(data.querySelectorAll('sitemap loc')).map(n => n.textContent);
return otherSitemaps.reduce( async (previousPromise, url) => {
await previousPromise;
return fetch(url).then(ress => ress.text())
.then(strr => new window.DOMParser().parseFromString(strr.trimStart(), "text/xml"))
.then(subdata => {
urls = urls.concat(Array.from(subdata.querySelectorAll('url loc')).map(n => n.textContent));
urls = urls.concat(Array.from(subdata.querySelectorAll('url [hreflang]')).map(n => n.getAttribute('href')));
});
}, Promise.resolve());
});
}
function removeStaticAssets(urls) {
const staticExtensions = ['.js', '.css', '.png', '.jpg', '.jpeg', '.gif', '.svg', '.webp', '.png', '.bmp', '.avif', '.mp4', '.mov', '.pdf'];
const filteredUrls = urls.filter(url => {
const extension = url.substring(url.lastIndexOf('.')).toLowerCase();
return !staticExtensions.includes(extension);
});
return filteredUrls;
}
await getUrls();
// remove duplicates
urls = [...new Set(urls)];
// remove static assetes
urls = removeStaticAssets(urls);
urls.reduce( async (previousPromise, url) => {
await previousPromise;
return fetch(url);
}, Promise.resolve());
}
crawl();
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment