feat: database

server.js

@@ -8,6 +8,10 @@ const fs = require('fs').promises;
 const path = require('path');
 const axios = require('axios');
 const cheerio = require('cheerio');
+const { Op } = require('sequelize');
+const db = require('./models');
+const MigrationRunner = require('./migrations/runner');
+const { v4: uuidv4 } = require('uuid');
 
 // Lazy load puppeteer (only if needed)
 let puppeteer = null;
@@ -190,54 +194,168 @@ function isAuthenticated(req, res, next) {
   res.status(401).json({ error: 'Authentication required' });
 }
 
-// Ensure data directory exists
-async function ensureDataDir() {
-  const dataDir = path.dirname(DATA_FILE);
+// Database initialization and migration
+async function initializeDatabase() {
   try {
-    await fs.access(dataDir);
-  } catch {
-    await fs.mkdir(dataDir, { recursive: true });
-  }
-  try {
-    await fs.access(DATA_FILE);
-  } catch {
-    await fs.writeFile(DATA_FILE, JSON.stringify([]));
-  }
-  try {
-    await fs.access(LISTS_FILE);
-  } catch {
-    await fs.writeFile(LISTS_FILE, JSON.stringify([]));
-  }
-}
+    // Test database connection
+    await db.sequelize.authenticate();
+    console.log('Database connection established successfully.');
 
-// Read links from file
-async function readLinks() {
-  try {
-    const data = await fs.readFile(DATA_FILE, 'utf8');
-    return JSON.parse(data);
+    // Run migrations
+    const migrationRunner = new MigrationRunner(db.sequelize);
+    await migrationRunner.runMigrations();
+
+    // Migrate JSON files if they exist
+    await migrateJsonFiles();
+
+    console.log('Database initialization completed.');
   } catch (error) {
-    return [];
+    console.error('Database initialization failed:', error);
+    throw error;
   }
 }
 
-// Write links to file
-async function writeLinks(links) {
-  await fs.writeFile(DATA_FILE, JSON.stringify(links, null, 2));
-}
+// Migrate JSON files to database
+async function migrateJsonFiles() {
+  const linksFile = DATA_FILE;
+  const listsFile = LISTS_FILE;
+  const linksBackup = linksFile + '.bak';
+  const listsBackup = listsFile + '.bak';
 
-// Read lists from file
-async function readLists() {
+  // Check if files have already been migrated
+  let linksAlreadyMigrated = false;
+  let listsAlreadyMigrated = false;
+  
   try {
-    const data = await fs.readFile(LISTS_FILE, 'utf8');
-    return JSON.parse(data);
-  } catch (error) {
-    return [];
+    await fs.access(linksBackup);
+    linksAlreadyMigrated = true;
+  } catch {
+    // Not migrated yet
+  }
+  
+  try {
+    await fs.access(listsBackup);
+    listsAlreadyMigrated = true;
+  } catch {
+    // Not migrated yet
+  }
+
+  // Step 1: Migrate lists first (so we can create relationships)
+  const listIdMap = new Map(); // Map old ID -> new UUID
+  
+  if (!listsAlreadyMigrated) {
+    try {
+      await fs.access(listsFile);
+      const listsData = JSON.parse(await fs.readFile(listsFile, 'utf8'));
+      
+      if (Array.isArray(listsData) && listsData.length > 0) {
+        console.log(`Migrating ${listsData.length} lists from JSON file...`);
+        
+        for (const list of listsData) {
+          const newId = uuidv4();
+          listIdMap.set(list.id, newId);
+          
+          await db.List.create({
+            id: newId,
+            name: list.name,
+            created_at: list.createdAt ? new Date(list.createdAt) : new Date(),
+            created_by: null, // No user info in JSON
+            public: list.public || false
+          });
+        }
+        
+        // Rename file to backup
+        await fs.rename(listsFile, listsBackup);
+        console.log('Lists migration completed.');
+      }
+    } catch (error) {
+      if (error.code !== 'ENOENT') {
+        console.error('Error migrating lists:', error);
+      }
+    }
+  }
+
+  // Step 2: Migrate links and set up relationships
+  if (!linksAlreadyMigrated) {
+    try {
+      await fs.access(linksFile);
+      const linksData = JSON.parse(await fs.readFile(linksFile, 'utf8'));
+      
+      if (Array.isArray(linksData) && linksData.length > 0) {
+        console.log(`Migrating ${linksData.length} links from JSON file...`);
+        
+        for (const link of linksData) {
+          // Create link
+          const linkRecord = await db.Link.create({
+            id: uuidv4(),
+            url: link.url,
+            title: link.title || null,
+            description: link.description || null,
+            image: link.image || null,
+            created_at: link.createdAt ? new Date(link.createdAt) : new Date(),
+            created_by: null, // No user info in JSON
+            archived: link.archived || false
+          });
+
+          // Create relationships if listIds exist
+          if (link.listIds && Array.isArray(link.listIds) && link.listIds.length > 0) {
+            const listRecords = [];
+            for (const oldListId of link.listIds) {
+              const newListId = listIdMap.get(oldListId);
+              if (newListId) {
+                const listRecord = await db.List.findByPk(newListId);
+                if (listRecord) {
+                  listRecords.push(listRecord);
+                }
+              }
+            }
+            if (listRecords.length > 0) {
+              await linkRecord.setLists(listRecords);
+            }
+          }
+        }
+        
+        // Rename file to backup
+        await fs.rename(linksFile, linksBackup);
+        console.log('Links migration completed.');
+      }
+    } catch (error) {
+      if (error.code !== 'ENOENT') {
+        console.error('Error migrating links:', error);
+      }
+    }
   }
 }
 
-// Write lists to file
-async function writeLists(lists) {
-  await fs.writeFile(LISTS_FILE, JSON.stringify(lists, null, 2));
+// Helper function to format link for API response
+function formatLink(link) {
+  const formatted = {
+    id: link.id,
+    url: link.url,
+    title: link.title,
+    description: link.description,
+    image: link.image,
+    createdAt: link.created_at,
+    createdBy: link.created_by,
+    modifiedAt: link.modified_at,
+    modifiedBy: link.modified_by,
+    archived: link.archived || false,
+    listIds: link.lists ? link.lists.map(list => list.id) : []
+  };
+  return formatted;
+}
+
+// Helper function to format list for API response
+function formatList(list) {
+  return {
+    id: list.id,
+    name: list.name,
+    createdAt: list.created_at,
+    createdBy: list.created_by,
+    modifiedAt: list.modified_at,
+    modifiedBy: list.modified_by,
+    public: list.public || false
+  };
 }
 
 // Extract metadata using Puppeteer (for JavaScript-heavy sites)
@@ -755,25 +873,42 @@ app.post('/api/auth/logout', (req, res) => {
 // Get all links
 app.get('/api/links', async (req, res) => {
   try {
-    const links = await readLinks();
+    let links;
     
     // If user is not authenticated, only show links in public lists
     if (!req.isAuthenticated()) {
-      const lists = await readLists();
-      const publicListIds = lists.filter(list => list.public === true).map(list => list.id);
-      
-      // Filter links to only those that are in at least one public list
-      const filteredLinks = links.filter(link => {
-        const linkListIds = link.listIds || [];
-        return linkListIds.some(listId => publicListIds.includes(listId));
+      // Get all public lists
+      const publicLists = await db.List.findAll({
+        where: { public: true }
       });
+      const publicListIds = publicLists.map(list => list.id);
       
-      return res.json(filteredLinks);
+      // Get links that are in at least one public list
+      links = await db.Link.findAll({
+        include: [{
+          model: db.List,
+          as: 'lists',
+          where: { id: { [Op.in]: publicListIds } },
+          required: true,
+          attributes: ['id']
+        }],
+        order: [['created_at', 'DESC']]
+      });
+    } else {
+      // Authenticated users see all links
+      links = await db.Link.findAll({
+        include: [{
+          model: db.List,
+          as: 'lists',
+          attributes: ['id']
+        }],
+        order: [['created_at', 'DESC']]
+      });
     }
     
-    // Authenticated users see all links
-    res.json(links);
+    res.json(links.map(formatLink));
   } catch (error) {
+    console.error('Error fetching links:', error);
     res.status(500).json({ error: 'Failed to read links' });
   }
 });
@@ -782,33 +917,51 @@ app.get('/api/links', async (req, res) => {
 app.get('/api/links/search', async (req, res) => {
   try {
     const query = req.query.q?.toLowerCase() || '';
-    let links = await readLinks();
+    
+    const whereClause = {};
+    if (query) {
+      whereClause[Op.or] = [
+        { title: { [Op.iLike]: `%${query}%` } },
+        { description: { [Op.iLike]: `%${query}%` } },
+        { url: { [Op.iLike]: `%${query}%` } }
+      ];
+    }
+    
+    let links;
     
     // If user is not authenticated, only show links in public lists
     if (!req.isAuthenticated()) {
-      const lists = await readLists();
-      const publicListIds = lists.filter(list => list.public === true).map(list => list.id);
+      const publicLists = await db.List.findAll({
+        where: { public: true }
+      });
+      const publicListIds = publicLists.map(list => list.id);
       
-      // Filter links to only those that are in at least one public list
-      links = links.filter(link => {
-        const linkListIds = link.listIds || [];
-        return linkListIds.some(listId => publicListIds.includes(listId));
+      links = await db.Link.findAll({
+        where: whereClause,
+        include: [{
+          model: db.List,
+          as: 'lists',
+          where: { id: { [Op.in]: publicListIds } },
+          required: true,
+          attributes: ['id']
+        }],
+        order: [['created_at', 'DESC']]
+      });
+    } else {
+      links = await db.Link.findAll({
+        where: whereClause,
+        include: [{
+          model: db.List,
+          as: 'lists',
+          attributes: ['id']
+        }],
+        order: [['created_at', 'DESC']]
       });
     }
     
-    if (!query) {
-      return res.json(links);
-    }
-    
-    const filtered = links.filter(link => {
-      const titleMatch = link.title?.toLowerCase().includes(query);
-      const descMatch = link.description?.toLowerCase().includes(query);
-      const urlMatch = link.url?.toLowerCase().includes(query);
-      return titleMatch || descMatch || urlMatch;
-    });
-    
-    res.json(filtered);
+    res.json(links.map(formatLink));
   } catch (error) {
+    console.error('Error searching links:', error);
     res.status(500).json({ error: 'Failed to search links' });
   }
 });
@@ -823,8 +976,7 @@ app.post('/api/links', isAuthenticated, async (req, res) => {
     }
     
     // Check if link already exists
-    const links = await readLinks();
-    const existingLink = links.find(link => link.url === url);
+    const existingLink = await db.Link.findOne({ where: { url } });
     if (existingLink) {
       return res.status(409).json({ error: 'Link already exists' });
     }
@@ -833,19 +985,19 @@ app.post('/api/links', isAuthenticated, async (req, res) => {
     const metadata = await extractMetadata(url);
     
     // Create new link
-    const newLink = {
-      id: Date.now().toString(),
+    const newLink = await db.Link.create({
       url: url,
       title: metadata.title,
       description: metadata.description,
       image: metadata.image,
-      createdAt: new Date().toISOString()
-    };
+      created_by: req.user?.username || null,
+      archived: false
+    });
     
-    links.unshift(newLink); // Add to beginning
-    await writeLinks(links);
+    // Reload with associations to get listIds
+    await newLink.reload({ include: [{ model: db.List, as: 'lists', attributes: ['id'] }] });
     
-    res.status(201).json(newLink);
+    res.status(201).json(formatLink(newLink));
   } catch (error) {
     console.error('Error adding link:', error);
     res.status(500).json({ error: 'Failed to add link' });
@@ -862,18 +1014,24 @@ app.patch('/api/links/:id/archive', isAuthenticated, async (req, res) => {
       return res.status(400).json({ error: 'archived must be a boolean' });
     }
     
-    const links = await readLinks();
-    const linkIndex = links.findIndex(link => link.id === id);
+    const link = await db.Link.findByPk(id, {
+      include: [{ model: db.List, as: 'lists', attributes: ['id'] }]
+    });
     
-    if (linkIndex === -1) {
+    if (!link) {
       return res.status(404).json({ error: 'Link not found' });
     }
     
-    links[linkIndex].archived = archived;
-    await writeLinks(links);
+    await link.update({
+      archived: archived,
+      modified_by: req.user?.username || null
+    });
     
-    res.json(links[linkIndex]);
+    await link.reload({ include: [{ model: db.List, as: 'lists', attributes: ['id'] }] });
+    
+    res.json(formatLink(link));
   } catch (error) {
+    console.error('Error updating link:', error);
     res.status(500).json({ error: 'Failed to update link' });
   }
 });
@@ -882,16 +1040,16 @@ app.patch('/api/links/:id/archive', isAuthenticated, async (req, res) => {
 app.delete('/api/links/:id', isAuthenticated, async (req, res) => {
   try {
     const { id } = req.params;
-    const links = await readLinks();
-    const filtered = links.filter(link => link.id !== id);
+    const link = await db.Link.findByPk(id);
     
-    if (filtered.length === links.length) {
+    if (!link) {
       return res.status(404).json({ error: 'Link not found' });
     }
     
-    await writeLinks(filtered);
+    await link.destroy();
     res.json({ message: 'Link deleted successfully' });
   } catch (error) {
+    console.error('Error deleting link:', error);
     res.status(500).json({ error: 'Failed to delete link' });
   }
 });
@@ -906,18 +1064,31 @@ app.patch('/api/links/:id/lists', isAuthenticated, async (req, res) => {
       return res.status(400).json({ error: 'listIds must be an array' });
     }
     
-    const links = await readLinks();
-    const linkIndex = links.findIndex(link => link.id === id);
+    const link = await db.Link.findByPk(id);
     
-    if (linkIndex === -1) {
+    if (!link) {
       return res.status(404).json({ error: 'Link not found' });
     }
     
-    links[linkIndex].listIds = listIds;
-    await writeLinks(links);
+    // Find all lists by IDs
+    const lists = await db.List.findAll({
+      where: { id: { [Op.in]: listIds } }
+    });
     
-    res.json(links[linkIndex]);
+    // Update relationships
+    await link.setLists(lists);
+    
+    // Update modified fields
+    await link.update({
+      modified_by: req.user?.username || null
+    });
+    
+    // Reload with associations
+    await link.reload({ include: [{ model: db.List, as: 'lists', attributes: ['id'] }] });
+    
+    res.json(formatLink(link));
   } catch (error) {
+    console.error('Error updating link lists:', error);
     res.status(500).json({ error: 'Failed to update link lists' });
   }
 });
@@ -927,17 +1098,24 @@ app.patch('/api/links/:id/lists', isAuthenticated, async (req, res) => {
 // Get all lists
 app.get('/api/lists', async (req, res) => {
   try {
-    const lists = await readLists();
+    let lists;
     
     // If user is not authenticated, only return public lists
     if (!req.isAuthenticated()) {
-      const publicLists = lists.filter(list => list.public === true);
-      return res.json(publicLists);
+      lists = await db.List.findAll({
+        where: { public: true },
+        order: [['created_at', 'DESC']]
+      });
+    } else {
+      // Authenticated users see all lists
+      lists = await db.List.findAll({
+        order: [['created_at', 'DESC']]
+      });
     }
     
-    // Authenticated users see all lists
-    res.json(lists);
+    res.json(lists.map(formatList));
   } catch (error) {
+    console.error('Error fetching lists:', error);
     res.status(500).json({ error: 'Failed to read lists' });
   }
 });
@@ -951,26 +1129,28 @@ app.post('/api/lists', isAuthenticated, async (req, res) => {
       return res.status(400).json({ error: 'List name is required' });
     }
     
-    const lists = await readLists();
+    const trimmedName = name.trim();
+    
+    // Check if list with same name already exists (case-insensitive)
+    const existingList = await db.List.findOne({
+      where: {
+        name: { [Op.iLike]: trimmedName }
+      }
+    });
     
-    // Check if list with same name already exists
-    const existingList = lists.find(list => list.name.toLowerCase() === name.trim().toLowerCase());
     if (existingList) {
       return res.status(409).json({ error: 'List with this name already exists' });
     }
     
-    const newList = {
-      id: Date.now().toString(),
-      name: name.trim(),
-      createdAt: new Date().toISOString(),
+    const newList = await db.List.create({
+      name: trimmedName,
+      created_by: req.user?.username || null,
       public: false
-    };
+    });
     
-    lists.push(newList);
-    await writeLists(lists);
-    
-    res.status(201).json(newList);
+    res.status(201).json(formatList(newList));
   } catch (error) {
+    console.error('Error creating list:', error);
     res.status(500).json({ error: 'Failed to create list' });
   }
 });
@@ -985,24 +1165,34 @@ app.put('/api/lists/:id', isAuthenticated, async (req, res) => {
       return res.status(400).json({ error: 'List name is required' });
     }
     
-    const lists = await readLists();
-    const listIndex = lists.findIndex(list => list.id === id);
+    const list = await db.List.findByPk(id);
     
-    if (listIndex === -1) {
+    if (!list) {
       return res.status(404).json({ error: 'List not found' });
     }
     
-    // Check if another list with same name exists
-    const existingList = lists.find(list => list.id !== id && list.name.toLowerCase() === name.trim().toLowerCase());
+    const trimmedName = name.trim();
+    
+    // Check if another list with same name exists (case-insensitive)
+    const existingList = await db.List.findOne({
+      where: {
+        id: { [Op.ne]: id },
+        name: { [Op.iLike]: trimmedName }
+      }
+    });
+    
     if (existingList) {
       return res.status(409).json({ error: 'List with this name already exists' });
     }
     
-    lists[listIndex].name = name.trim();
-    await writeLists(lists);
+    await list.update({
+      name: trimmedName,
+      modified_by: req.user?.username || null
+    });
     
-    res.json(lists[listIndex]);
+    res.json(formatList(list));
   } catch (error) {
+    console.error('Error updating list:', error);
     res.status(500).json({ error: 'Failed to update list' });
   }
 });
@@ -1017,18 +1207,20 @@ app.patch('/api/lists/:id/public', isAuthenticated, async (req, res) => {
       return res.status(400).json({ error: 'public must be a boolean' });
     }
     
-    const lists = await readLists();
-    const listIndex = lists.findIndex(list => list.id === id);
+    const list = await db.List.findByPk(id);
     
-    if (listIndex === -1) {
+    if (!list) {
       return res.status(404).json({ error: 'List not found' });
     }
     
-    lists[listIndex].public = isPublic;
-    await writeLists(lists);
+    await list.update({
+      public: isPublic,
+      modified_by: req.user?.username || null
+    });
     
-    res.json(lists[listIndex]);
+    res.json(formatList(list));
   } catch (error) {
+    console.error('Error updating list public status:', error);
     res.status(500).json({ error: 'Failed to update list public status' });
   }
 });
@@ -1037,25 +1229,18 @@ app.patch('/api/lists/:id/public', isAuthenticated, async (req, res) => {
 app.delete('/api/lists/:id', isAuthenticated, async (req, res) => {
   try {
     const { id } = req.params;
-    const lists = await readLists();
-    const filtered = lists.filter(list => list.id !== id);
+    const list = await db.List.findByPk(id);
     
-    if (filtered.length === lists.length) {
+    if (!list) {
       return res.status(404).json({ error: 'List not found' });
     }
     
-    // Remove this list from all links
-    const links = await readLinks();
-    links.forEach(link => {
-      if (link.listIds && Array.isArray(link.listIds)) {
-        link.listIds = link.listIds.filter(listId => listId !== id);
-      }
-    });
-    await writeLinks(links);
+    // CASCADE delete will automatically remove from link_lists junction table
+    await list.destroy();
     
-    await writeLists(filtered);
     res.json({ message: 'List deleted successfully' });
   } catch (error) {
+    console.error('Error deleting list:', error);
     res.status(500).json({ error: 'Failed to delete list' });
   }
 });
@@ -1072,10 +1257,18 @@ function isValidUrl(string) {
 
 // Initialize server
 async function startServer() {
-  await ensureDataDir();
-  app.listen(PORT, () => {
-    console.log(`LinkDing server running on http://localhost:${PORT}`);
-  });
+  try {
+    // Initialize database (connect, run migrations, migrate JSON files)
+    await initializeDatabase();
+    
+    // Start server
+    app.listen(PORT, () => {
+      console.log(`LinkDing server running on http://localhost:${PORT}`);
+    });
+  } catch (error) {
+    console.error('Failed to start server:', error);
+    process.exit(1);
+  }
 }
 
 startServer();