@@ -10,6 +10,7 @@
 
 # Misc
 .DS_Store
+.env
 .env.local
 .env.development.local
 .env.test.local
@@ -18,3 +19,6 @@
 npm-debug.log*
 yarn-debug.log*
 yarn-error.log*
+
+# Algolia api key
+.algolia-apikey.js
@@ -0,0 +1,111 @@
+/* ALGOLIA SCRAPER
+ * This script searches for relevant content to index in docsDir.
+ * Before running it make sure you have the file algolia-apikey.js
+ * containing your ALGOLIA_API_KEY and make sure you have it on gitignore.
+ *
+ * example: node algolia.js
+ *
+ * IMPORTANT NOTE:
+ * We have been accepted into the algolia program for FOSS projects.
+ * So this scraper is not really necessary anymore unless that changes.
+ * */
+
+const fs = require('fs');
+const path = require('path');
+const algoliasearch = require('algoliasearch');
+const cheerio = require('cheerio');
+const { ALGOLIA_API_KEY } = require('./.algolia-apikey');
+
+// MAKE SURE YOUR TOKEN IS CORRECT
+// Initialize Algolia client with your credentials
+const appId = 'S1DLXCNPKJ'; // Your Algolia Application ID
+const apiKey = ALGOLIA_API_KEY; // Get API key from apikey.js
+const client = algoliasearch(appId, apiKey);
+const index = client.initIndex('normalnvim');
+
+// Directory containing your Docusaurus build content
+const docsDir = path.join(__dirname, 'build', 'docs');
+
+// Toggle DEBUG_MODE to true or false
+const DEBUG_MODE = false;
+
+// Debug print function
+function debugPrint(message, ...optionalParams) {
+  if (DEBUG_MODE) {
+    console.log(message, ...optionalParams);
+  }
+}
+
+// Function to calculate relevancy score
+function calculateRelevancyScore(content) {
+  // Basic example: higher scores for longer content
+  // You can enhance this function with more complex logic
+  const lengthScore = content.length / 1000; // Normalize length
+  return Math.min(lengthScore, 1); // Ensure score is between 0 and 1
+}
+
+// Recursively traverse directories and process files
+function extractDocsFromDir(dir) {
+  const records = [];
+
+  function readDirRecursive(currentDir) {
+    const files = fs.readdirSync(currentDir);
+
+    files.forEach((file) => {
+      const filePath = path.join(currentDir, file);
+      const stats = fs.statSync(filePath);
+
+      if (stats.isDirectory()) {
+        // Recursively process subdirectories
+        readDirRecursive(filePath);
+      } else if (stats.isFile() && file.endsWith('.html')) {
+        // Process HTML files
+        debugPrint(`Processing file: ${filePath}`);
+        const content = fs.readFileSync(filePath, 'utf8');
+        debugPrint(`Content of ${file}:`, content.slice(0, 100)); // Show first 100 characters
+
+        // Parse and create records using cheerio
+        const $ = cheerio.load(content);
+        const title = $('h1').first().text() || $('title').text() || '';
+        const bodyContent = $('main').text() || $('body').text() || '';
+
+        const relevancyScore = calculateRelevancyScore(bodyContent);
+
+        const record = {
+          objectID: path.relative(docsDir, filePath),
+          title: title,
+          content: bodyContent,
+          url: `https://normalnvim.github.io/docs/${path.relative(docsDir, filePath).replace(/\index.html$/, '')}`,
+          relevancy: relevancyScore,
+          hierarchy: {
+            lvl0: title || '',
+            lvl1: '', // Adjust if your content has specific sub-levels
+            lvl2: '', // Adjust if needed
+            lvl3: '', // Adjust if needed
+            lvl4: '', // Adjust if needed
+            lvl5: '', // Adjust if needed
+          },
+        };
+
+        debugPrint('Generated record:', record);
+        records.push(record);
+      }
+    });
+  }
+
+  readDirRecursive(dir);
+  return records;
+}
+
+async function uploadRecords() {
+  const records = extractDocsFromDir(docsDir);
+  try {
+    const { objectIDs } = await index.saveObjects(records);
+    console.log('Algolia - Index updated successfully:', objectIDs);
+  } catch (error) {
+    console.error('Error uploading documents:', error);
+  }
+}
+
+uploadRecords();
+