feat: exclude devto and hashnode posts that first appeared on networkandcode.github.io

Author: networkandcode

src/data/cv.json

@@ -653,73 +653,5 @@
     "reading_time_minutes": 5,
     "positive_reactions_count": 10,
     "comments_count": 2
-  },
-  {
-    "id": 917875,
-    "title": "Setup an IAM user in AWS via CLI",
-    "description": "Hey all, 👋 we shall see the following in this post: 📜   Create a group Attach policies ✍️ to the...",
-    "url": "https://dev.to/aws-builders/aws-iam-setup-user-via-cli-45bb",
-    "published_at": "2021-12-05T11:19:02Z",
-    "tag_list": [
-      "aws",
-      "iam",
-      "jq",
-      "security"
-    ],
-    "cover_image": null,
-    "reading_time_minutes": 4,
-    "positive_reactions_count": 7,
-    "comments_count": 0
-  },
-  {
-    "id": 897816,
-    "title": "Launch AWS EC2 instances the hard way with CLI",
-    "description": "Hey All 👋, in this post we shall launch 3 AWS EC2 instances and test SSH connectivity to those...  We...",
-    "url": "https://dev.to/aws-builders/aws-ec2-launch-instances-the-hard-way-with-cli-2ga3",
-    "published_at": "2021-11-14T12:11:43Z",
-    "tag_list": [
-      "aws",
-      "ec2",
-      "linux",
-      "kubernetes"
-    ],
-    "cover_image": null,
-    "reading_time_minutes": 6,
-    "positive_reactions_count": 6,
-    "comments_count": 0
-  },
-  {
-    "id": 865633,
-    "title": "Setup an AWS EKS cluster and deploy OWASP juice shop",
-    "description": "Let's setup an EKS cluster and then install the  OWASP Juice Shop on it using kubernetes manifests.  ...",
-    "url": "https://dev.to/aws-builders/aws-eks-setup-cluster-and-deploy-owasp-juice-shop-214j",
-    "published_at": "2021-10-16T09:00:00Z",
-    "tag_list": [
-      "kubernetes",
-      "aws",
-      "owasp",
-      "eks"
-    ],
-    "cover_image": null,
-    "reading_time_minutes": 6,
-    "positive_reactions_count": 7,
-    "comments_count": 0
-  },
-  {
-    "id": 863227,
-    "title": "Bootstrap a Kubernetes cluster with RKE",
-    "description": "Rancher Kubernetes Engine ( RKE ) is a popular tool using which we can bootstrap kubernetes clusters...",
-    "url": "https://dev.to/networkandcode/kubernetes-bootstrap-a-cluster-with-rke-1f9n",
-    "published_at": "2021-10-14T07:56:52Z",
-    "tag_list": [
-      "kubernetes",
-      "rancher",
-      "linode",
-      "linux"
-    ],
-    "cover_image": null,
-    "reading_time_minutes": 4,
-    "positive_reactions_count": 5,
-    "comments_count": 0
   }
 ]

src/data/devtoArticles.json

@@ -27,13 +27,6 @@
     "published_at": "2022-06-11",
     "description": "Introduction Appwrite is a backend server that can be leveraged for integrating services such as authentication, storage, database etc. to our frontend application. It can be launc..."
   },
-  {
-    "id": "https://networkandcode.hashnode.dev/run-owasp-juice-shop-as-a-kubernetes-service",
-    "title": "Run OWASP juice shop as a Kubernetes service",
-    "url": "https://networkandcode.hashnode.dev/run-owasp-juice-shop-as-a-kubernetes-service",
-    "published_at": "2021-08-07",
-    "description": "This article first appeared on https://networkandcode.github.io/owasp/juice/shop/2021/08/08/owasp-juice-shop-run-as-kubernetes-service.html Hello, we shall run the OWASP juice shop..."
-  },
   {
     "id": "https://networkandcode.hashnode.dev/online-shop-with-nextjs-and-harperdb",
     "title": "Online shop with NextJS and HarperDB",

src/data/hashnodeArticles.json

@@ -44,18 +44,25 @@ export function syncDevToPosts(): AstroIntegration {
           const dataFilePath = path.resolve(process.cwd(), 'src/data/devtoArticles.json');
 
           if (Array.isArray(articles) && articles.length > 0) {
-            const formatted: DevToArticle[] = articles.map((art: any) => ({
-              id: art.id,
-              title: art.title,
-              description: art.description || '',
-              url: art.url,
-              published_at: art.published_at,
-              tag_list: art.tag_list || [],
-              cover_image: art.cover_image || null,
-              reading_time_minutes: art.reading_time_minutes,
-              positive_reactions_count: art.positive_reactions_count,
-              comments_count: art.comments_count,
-            }));
+            const formatted: DevToArticle[] = articles
+              .filter((art: any) => {
+                const desc = (art.description || '').toLowerCase();
+                const canon = (art.canonical_url || '').toLowerCase();
+                // Exclude posts that first appeared on networkandcode.github.io
+                return !desc.includes('networkandcode.github.io') && !canon.includes('networkandcode.github.io');
+              })
+              .map((art: any) => ({
+                id: art.id,
+                title: art.title,
+                description: art.description || '',
+                url: art.url,
+                published_at: art.published_at,
+                tag_list: art.tag_list || [],
+                cover_image: art.cover_image || null,
+                reading_time_minutes: art.reading_time_minutes,
+                positive_reactions_count: art.positive_reactions_count,
+                comments_count: art.comments_count,
+              }));
 
             fs.writeFileSync(dataFilePath, JSON.stringify(formatted, null, 2), 'utf-8');
             console.log(`[sync-devto] Synced ${formatted.length} DEV.to articles metadata to src/data/devtoArticles.json`);

src/lib/syncDevTo.ts

@@ -46,11 +46,18 @@ export function syncHashnodePosts(): AstroIntegration {
             const descMatch = itemContent.match(/<description><!\[CDATA\[([\s\S]*?)\]\]><\/description>/);
             const guidMatch = itemContent.match(/<guid[^>]*>(.*?)<\/guid>/);
 
-            let description = '';
+            let rawDescription = '';
             if (descMatch) {
-              description = descMatch[1].replace(/<[^>]+>/g, '').replace(/\s+/g, ' ').trim().slice(0, 180) + '...';
+              rawDescription = descMatch[1].replace(/<[^>]+>/g, '').replace(/\s+/g, ' ').trim();
             }
 
+            // Rule: Exclude posts that first appeared on networkandcode.github.io
+            if (rawDescription.toLowerCase().includes('networkandcode.github.io') || rawDescription.toLowerCase().includes('first appeared on https://networkandcode')) {
+              continue;
+            }
+
+            const description = rawDescription.length > 180 ? rawDescription.slice(0, 180) + '...' : rawDescription;
+
             if (titleMatch && linkMatch) {
               items.push({
                 id: guidMatch ? guidMatch[1] : linkMatch[1],

src/lib/syncHashnode.ts

@@ -58,7 +58,13 @@ const processedLocalPosts: PostItem[] = localPosts.map((post) => {
   };
 });
 
-const processedDevToPosts: PostItem[] = (devtoData as any[]).map((art) => {
+const processedDevToPosts: PostItem[] = (devtoData as any[])
+  .filter((art) => {
+    const desc = (art.description || '').toLowerCase();
+    const url = (art.url || '').toLowerCase();
+    return !desc.includes('networkandcode.github.io') && !url.includes('networkandcode.github.io');
+  })
+  .map((art) => {
   const dateStr = art.published_at ? art.published_at.slice(0, 10) : '2024-01-01';
   const tagsStr = (art.tag_list || []).join(' ').toLowerCase();
   const textStr = (art.title + ' ' + tagsStr).toLowerCase();
@@ -89,7 +95,13 @@ const processedDevToPosts: PostItem[] = (devtoData as any[]).map((art) => {
   };
 });
 
-const processedHashnodePosts: PostItem[] = (hashnodeData as any[]).map((art) => {
+const processedHashnodePosts: PostItem[] = (hashnodeData as any[])
+  .filter((art) => {
+    const desc = (art.description || '').toLowerCase();
+    const url = (art.url || '').toLowerCase();
+    return !desc.includes('networkandcode.github.io') && !url.includes('networkandcode.github.io');
+  })
+  .map((art) => {
   const dateStr = art.published_at ? art.published_at.slice(0, 10) : '2022-06-01';
   const textStr = (art.title + ' ' + (art.description || '')).toLowerCase();
 

src/pages/blog/index.astro

@@ -35,7 +35,9 @@ const coreCompetencies = [
   }
 ];
 
-const totalPosts = posts.length + (devtoData as any[]).length + (hashnodeData as any[]).length;
+const validDevTo = (devtoData as any[]).filter(a => !((a.description || '').toLowerCase().includes('networkandcode.github.io')));
+const validHashnode = (hashnodeData as any[]).filter(a => !((a.description || '').toLowerCase().includes('networkandcode.github.io')));
+const totalPosts = posts.length + validDevTo.length + validHashnode.length;
 const totalVideos = youtubeVideos.length;
 const totalCerts = accreditations.length;
 const totalAwards = awardsAndCertificates.length;