How to Hide Pages from Search Engines and Sitemaps in Sanity CMS: A Complete Guide

import { DocumentIcon } from '@sanity/icons';import type { Rule } from 'sanity';export const blog = {  name: 'blog',  title: 'Blog Post',  type: 'document',  icon: DocumentIcon,  fields: [    {      name: 'title',      title: 'Title',      type: 'string',      validation: (rule: Rule) => rule.required(),    },    {      name: 'slug',      title: 'Slug',      type: 'slug',      options: {        source: 'title',        maxLength: 96,      },      validation: (rule: Rule) => rule.required(),    },    {      name: 'hideFromSearch?',      title: 'Hide from Search Engines',      type: 'boolean',      description: 'When enabled, this page will not appear in search engines, sitemaps, or public listings. Use for testing, private content, or draft materials.',      initialValue: false,    },    // ... other fields    {      name: 'content',      title: 'Content',      type: 'array',      of: [{ type: 'block' }],    },  ],  preview: {    select: {      title: 'title',      hideFromSearch: 'hideFromSearch',    },    prepare(selection) {      const { title, hideFromSearch } = selection;      return {        title: title,        subtitle: hideFromSearch ? '🔒 Hidden from search' : '🌐 Public',      };    },  },};

// Enhanced schema with additional controls{  name: 'seoSettings',  title: 'SEO Settings',  type: 'object',  fields: [    {      name: 'hideFromSearch',      title: 'Hide from Search Engines?',      type: 'boolean',      description: 'Excludes this page from search engines, sitemaps, and public listings.',      initialValue: false,    },    {      name: 'hideReason',      title: 'Reason for Hiding',      type: 'string',      options: {        list: [          { title: 'Testing/Draft Content', value: 'testing' },          { title: 'Client-Specific Content', value: 'client' },          { title: 'Seasonal/Temporary', value: 'seasonal' },          { title: 'Internal Use Only', value: 'internal' },        ],      },      hidden: ({ parent }) => !parent?.hideFromSearch,      validation: (rule: Rule) =>         rule.custom((value, context) => {          const parent = context.parent as { hideFromSearch?: boolean };          if (parent?.hideFromSearch && !value) {            return 'Please specify why this content is hidden';          }          return true;        }),    },  ],}

export const visibilityField = {  name: 'hideFromSearch',  title: 'Hide from Search Engines',  type: 'boolean',  description: 'When enabled, this content will be excluded from search engines, sitemaps, and public listings',  initialValue: false,  group: 'seo', // If you're using field groups};// Usage in any schemaimport { visibilityField } from '../fields/visibility';export const caseStudy = {  name: 'caseStudy',  title: 'Case Study',  type: 'document',  fields: [    // ... other fields    visibilityField,  ],};

import type { MetadataRoute } from 'next';import { client } from '~/lib/sanity';type ContentItem = {  slug: string;  _updatedAt: string;};export default async function sitemap(): Promise<MetadataRoute.Sitemap> {  const baseUrl = 'https://yourdomain.com';    // Static pages  const staticPages = [    {      url: baseUrl,      lastModified: new Date(),      changeFrequency: 'daily' as const,      priority: 1,    },    {      url: `${baseUrl}/about`,      lastModified: new Date(),      changeFrequency: 'monthly' as const,      priority: 0.8,    },    {      url: `${baseUrl}/blog`,      lastModified: new Date(),      changeFrequency: 'daily' as const,      priority: 0.8,    },  ];  // Dynamic content from Sanity  const dynamicPages = await getDynamicPages(baseUrl);  return [...staticPages, ...dynamicPages];}async function getDynamicPages(baseUrl: string): Promise<MetadataRoute.Sitemap> {  try {    const query = `{      "blogPosts": *[_type == "blog" && defined(slug.current) && hideFromSearch != true] {        "slug": slug.current,        _updatedAt      },      "caseStudies": *[_type == "caseStudy" && defined(slug.current) && hideFromSearch != true] {        "slug": slug.current,        _updatedAt      }    }`;    const { blogPosts, caseStudies } = await client.fetch<{      blogPosts: ContentItem[];      caseStudies: ContentItem[];    }>(query);    const dynamicPages: MetadataRoute.Sitemap = [];    // Add blog posts    blogPosts.forEach((post) => {      dynamicPages.push({        url: `${baseUrl}/blog/${post.slug}`,        lastModified: new Date(post._updatedAt),        changeFrequency: 'weekly',        priority: 0.6,      });    });    // Add case studies    caseStudies.forEach((study) => {      dynamicPages.push({        url: `${baseUrl}/case-studies/${study.slug}`,        lastModified: new Date(study._updatedAt),        changeFrequency: 'monthly',        priority: 0.7,      });    });    return dynamicPages;  } catch (error) {    console.error('Error generating dynamic sitemap:', error);    return [];  }}

import type { Metadata, ResolvingMetadata } from 'next';import { notFound } from 'next/navigation';import { client } from '~/lib/sanity';interface BlogPost {  title: string;  description: string;  hideFromSearch: boolean;  slug: string;  author?: {    name: string;  };  image?: {    url: string;    alt: string;  };}interface Props {  params: { slug: string };}async function getBlogPost(slug: string): Promise<BlogPost | null> {  const query = `    *[_type == "blog" && slug.current == $slug][0] {      title,      description,      hideFromSearch,      "slug": slug.current,      "author": author->{name},      "image": {        "url": image.asset->url,        "alt": image.alt      }    }  `;  return client.fetch(query, { slug });}export async function generateMetadata(  { params }: Props,  parent: ResolvingMetadata): Promise<Metadata> {  const post = await getBlogPost(params.slug);    if (!post) {    return {      title: 'Post Not Found',      robots: {        index: false,        follow: false,      },    };  }  const previousImages = (await parent).openGraph?.images || [];    const baseMetadata: Metadata = {    title: post.title,    description: post.description,    authors: post.author?.name ? [{ name: post.author.name }] : undefined,    openGraph: {      title: post.title,      description: post.description,      type: 'article',      images: post.image?.url ? [post.image.url, ...previousImages] : previousImages,    },  };  // Critical: Set robots meta tags for hidden content  if (post.hideFromSearch) {    baseMetadata.robots = {      index: false,      follow: false,    };  }  return baseMetadata;}export default async function BlogPostPage({ params }: Props) {  const post = await getBlogPost(params.slug);    if (!post?._id) {    return notFound();  }  return (    <article>      <h1>{post.title}</h1>      {/* Rest of your blog post content */}    </article>  );}

import { cache } from 'react';import { client } from '~/lib/sanity';interface BlogPost {  _id: string;  title: string;  slug: string;  description: string;  publishedAt: string;  author: {    name: string;    slug: string;  };  image?: {    url: string;    alt: string;  };}interface UseBlogPostsOptions {  limit?: number;  offset?: number;  authorId?: string;  category?: string;}const getBlogPosts = cache(async (options: UseBlogPostsOptions = {}) => {  const { limit = 10, offset = 0, authorId, category } = options;  // Build dynamic filters  let filters = ['_type == "blog"', 'hideFromSearch != true', 'defined(slug.current)'];    if (authorId) {    filters.push('references($authorId)');  }    if (category) {    filters.push('$category in categories[]->slug.current');  }  const filterString = filters.join(' && ');  const query = `{    "posts": *[${filterString}] | order(publishedAt desc) [${offset}...${offset + limit}] {      _id,      title,      "slug": slug.current,      description,      publishedAt,      "author": author->{        name,        "slug": slug.current      },      "image": {        "url": image.asset->url,        "alt": image.alt      }    },    "total": count(*[${filterString}])  }`;  const params: Record<string, any> = {};  if (authorId) params.authorId = authorId;  if (category) params.category = category;  return client.fetch<{    posts: BlogPost[];    total: number;  }>(query, params);});export async function useBlogPosts(options: UseBlogPostsOptions = {}) {  return getBlogPosts(options);}