> ## Documentation Index
> Fetch the complete documentation index at: https://hellotars.com/docs/llms.txt
> Use this file to discover all available pages before exploring further.

# Import a website

> Import selected website pages into a knowledge base, choose fetch settings, and keep the content fresh on a schedule.

export const Browser = ({url, children}) => <div className="tars-browser">
    <div className="tars-browser-bar">
      <span className="tars-browser-dots">
        <i />
        <i />
        <i />
      </span>
      {url ? <span className="tars-browser-url">{url}</span> : null}
    </div>
    <div className="tars-browser-view">{children}</div>
  </div>;

export const img = {
  activity: {
    activityTimeline: "/docs/guides/activity/images/activity-timeline.png",
    capturedVariablesPanel: "/docs/guides/activity/images/captured-variables-panel.png",
    dataViewCards: "/docs/guides/activity/images/data-view-cards.png",
    exportDataDialog: "/docs/guides/activity/images/export-data-dialog.png",
    tasksPanelExportReady: "/docs/guides/activity/images/tasks-panel-export-ready.png"
  },
  analytics: {
    analyticsOverview: "/docs/guides/analytics/images/analytics-overview.png",
    exploreGallery: "/docs/guides/analytics/images/explore-gallery.png",
    exploreTab: "/docs/guides/analytics/images/explore-tab.png",
    goalForm: "/docs/guides/analytics/images/goal-form.png",
    goalsTab: "/docs/guides/analytics/images/goals-tab.png",
    improveDiagnostics: "/docs/guides/analytics/images/improve-diagnostics.png",
    improveTab: "/docs/guides/analytics/images/improve-tab.png",
    teamTab: "/docs/guides/analytics/images/team-tab.png"
  },
  billing: {
    billingOverview: "/docs/guides/billing/images/billing-overview.png",
    buyConversationsDialog: "/docs/guides/billing/images/buy-conversations-dialog.png",
    buySeatsDialog: "/docs/guides/billing/images/buy-seats-dialog.png",
    livechatSeatsTab: "/docs/guides/billing/images/livechat-seats-tab.png",
    organizationDeletionLifecycle: "/docs/guides/billing/images/organization-deletion-lifecycle.svg",
    organizationSwitcher: "/docs/guides/billing/images/organization-switcher.png",
    rolesAndAccessMatrix: "/docs/guides/billing/images/roles-and-access-matrix.png"
  },
  building: {
    agentBehaviorGuardrails: "/docs/guides/building/images/agent-behavior-guardrails.png",
    agentCardMenu: "/docs/guides/building/images/agent-card-menu.png",
    agentConfigure: "/docs/guides/building/images/agent-configure.png",
    agentExitRoutesDialog: "/docs/guides/building/images/agent-exit-routes-dialog.png",
    agentLibrary: "/docs/guides/building/images/agent-library.png",
    agentModelParameters: "/docs/guides/building/images/agent-model-parameters.png",
    aiAgentGambitLoop: "/docs/guides/building/images/ai-agent-gambit-loop.svg",
    builderBasics: "/docs/guides/building/images/builder-basics.png",
    builderConnectHandlePanel: "/docs/guides/building/images/builder-connect-handle-panel.png",
    builtInToolsDialog: "/docs/guides/building/images/built-in-tools-dialog.png",
    configureDebugTab: "/docs/guides/building/images/configure-debug-tab.png",
    configureLanguage: "/docs/guides/building/images/configure-language.png",
    configureNotifications: "/docs/guides/building/images/configure-notifications.png",
    createAgentDialog: "/docs/guides/building/images/create-agent-dialog.png",
    draftPublishRollbackPinning: "/docs/guides/building/images/draft-publish-rollback-pinning.svg",
    draftSwitcherChip: "/docs/guides/building/images/draft-switcher-chip.png",
    duplicateAgentDialog: "/docs/guides/building/images/duplicate-agent-dialog.png",
    flowConditions: "/docs/guides/building/images/flow-conditions.png",
    gambitPalette: "/docs/guides/building/images/gambit-palette.png",
    gambits: {
      gambitAiAgent: "/docs/guides/building/gambits/images/gambit-ai-agent.png",
      gambitApiCall: "/docs/guides/building/gambits/images/gambit-api-call.png",
      gambitAutoSuggestion: "/docs/guides/building/gambits/images/gambit-auto-suggestion.png",
      gambitButton: "/docs/guides/building/gambits/images/gambit-button.png",
      gambitCard: "/docs/guides/building/gambits/images/gambit-card.png",
      gambitChannel: "/docs/guides/building/gambits/images/gambit-channel.png",
      gambitCustomCode: "/docs/guides/building/gambits/images/gambit-custom-code.png",
      gambitDateTime: "/docs/guides/building/gambits/images/gambit-date-time.png",
      gambitGeoLocation: "/docs/guides/building/gambits/images/gambit-geo-location.png",
      gambitLanguage: "/docs/guides/building/gambits/images/gambit-language.png",
      gambitLiveChat: "/docs/guides/building/gambits/images/gambit-live-chat.png",
      gambitMediaUpload: "/docs/guides/building/gambits/images/gambit-media-upload.png",
      gambitNoInput: "/docs/guides/building/gambits/images/gambit-no-input.png",
      gambitRedirect: "/docs/guides/building/gambits/images/gambit-redirect.png",
      gambitStarRating: "/docs/guides/building/gambits/images/gambit-star-rating.png",
      gambitStartAndEnd: "/docs/guides/building/gambits/images/gambit-start-and-end.png",
      gambitText: "/docs/guides/building/gambits/images/gambit-text.png",
      gambitTrigger: "/docs/guides/building/gambits/images/gambit-trigger.png"
    },
    languagesTranslateGambit: "/docs/guides/building/images/languages-translate-gambit.png",
    languageSystemMessages: "/docs/guides/building/images/language-system-messages.png",
    newDraftDialog: "/docs/guides/building/images/new-draft-dialog.png",
    outputComponentsGrid: "/docs/guides/building/images/output-components-grid.png",
    publishDialog: "/docs/guides/building/images/publish-dialog.png",
    systemPromptDialog: "/docs/guides/building/images/system-prompt-dialog.png",
    testModeEventLog: "/docs/guides/building/images/test-mode-event-log.png",
    testModePanel: "/docs/guides/building/images/test-mode-panel.png",
    validationProblemsPanel: "/docs/guides/building/images/validation-problems-panel.png",
    variablePickerPanel: "/docs/guides/building/images/variable-picker-panel.png",
    variablePillInserted: "/docs/guides/building/images/variable-pill-inserted.png",
    versionsDraftsPanel: "/docs/guides/building/images/versions-drafts-panel.png"
  },
  campaigns: {
    campaignFlavorsTriggers: "/docs/guides/campaigns/images/campaign-flavors-triggers.svg",
    campaignReplyRouting: "/docs/guides/campaigns/images/campaign-reply-routing.svg",
    campaignSendAnatomy: "/docs/guides/campaigns/images/campaign-send-anatomy.svg",
    createACampaign: "/docs/guides/campaigns/images/create-a-campaign.png",
    walletReserveSettleRefund: "/docs/guides/campaigns/images/wallet-reserve-settle-refund.svg",
    webhookCampaignContractLocking: "/docs/guides/campaigns/images/webhook-campaign-contract-locking.svg",
    webhookCampaignLifecycleStates: "/docs/guides/campaigns/images/webhook-campaign-lifecycle-states.svg",
    webhookCampaignOneCallOneDelivery: "/docs/guides/campaigns/images/webhook-campaign-one-call-one-delivery.svg"
  },
  channels: {
    distributeWhatsappConnect: "/docs/guides/channels/images/distribute-whatsapp-connect.png",
    whatsappConnectDialogWaiting: "/docs/guides/channels/images/whatsapp-connect-dialog-waiting.png",
    whatsappEmptyState: "/docs/guides/channels/images/whatsapp-empty-state.png"
  },
  deploy: {
    bridgeEventGambitSection: "/docs/guides/deploy/images/bridge-event-gambit-section.png",
    deploymentSurfacesAndChannels: "/docs/guides/deploy/images/deployment-surfaces-and-channels.svg",
    designAgentTab: "/docs/guides/deploy/images/design-agent-tab.png",
    designLauncherTab: "/docs/guides/deploy/images/design-launcher-tab.png",
    embedIframeTab: "/docs/guides/deploy/images/embed-iframe-tab.png",
    embedTheWidget: "/docs/guides/deploy/images/embed-the-widget.png",
    mobileAppIntegration: "/docs/guides/deploy/images/mobile-app-integration.png",
    onYourDomain: "/docs/guides/deploy/images/on-your-domain.png",
    shareYourAgentLink: "/docs/guides/deploy/images/share-your-agent-link.png"
  },
  developer: {
    api: {
      channelSendTemplateVsMessage: "/docs/developer/api/images/channel-send-template-vs-message.svg",
      endUsers: {
        endUsersListCursorPagination: "/docs/developer/api/end-users/images/end-users-list-cursor-pagination.svg"
      },
      triggerCallConversationLifecycle: "/docs/developer/api/images/trigger-call-conversation-lifecycle.svg"
    },
    developerSurfacesMap: "/docs/developer/images/developer-surfaces-map.svg",
    embed: {
      embedConfigToWidgetContext: "/docs/developer/embed/images/embed-config-to-widget-context.svg",
      hostPageBridgeFlow: "/docs/developer/embed/images/host-page-bridge-flow.svg",
      identitySigningFlow: "/docs/developer/embed/images/identity-signing-flow.svg",
      scriptVsIframeEmbed: "/docs/developer/embed/images/script-vs-iframe-embed.svg"
    },
    mobile: {
      bridgeMessageProtocol: "/docs/developer/mobile/images/bridge-message-protocol.svg",
      webviewBridgeArchitecture: "/docs/developer/mobile/images/webview-bridge-architecture.svg"
    }
  },
  endUsers: {
    anonymousToIdentifiedProfile: "/docs/guides/end-users/images/anonymous-to-identified-profile.svg",
    bulkActionBar: "/docs/guides/end-users/images/bulk-action-bar.png",
    campaignConsentGate: "/docs/guides/end-users/images/campaign-consent-gate.svg",
    consentPerChannelStates: "/docs/guides/end-users/images/consent-per-channel-states.svg",
    dynamicVsStaticSegments: "/docs/guides/end-users/images/dynamic-vs-static-segments.svg",
    endUserDirectory: "/docs/guides/end-users/images/end-user-directory.png",
    endUserProfileAcrossChannels: "/docs/guides/end-users/images/end-user-profile-across-channels.svg",
    endUserProfilePanes: "/docs/guides/end-users/images/end-user-profile-panes.png",
    profileMergeAndTakeoverProtection: "/docs/guides/end-users/images/profile-merge-and-takeover-protection.svg",
    segmentBuilder: "/docs/guides/end-users/images/segment-builder.png",
    segmentsPage: "/docs/guides/end-users/images/segments-page.png"
  },
  getStarted: {
    agentStates: "/docs/get-started/images/agent-states.svg",
    buildToLivePath: "/docs/get-started/images/build-to-live-path.svg",
    dashboardSidebarMap: "/docs/get-started/images/dashboard-sidebar-map.png",
    organizationOwnershipAndRoles: "/docs/get-started/images/organization-ownership-and-roles.svg",
    quickstart: {
      step1CreateYourAccount: "/docs/get-started/images/quickstart/step-1-create-your-account.png",
      step3ConnectTheAiAgent: "/docs/get-started/images/quickstart/step-3-connect-the-ai-agent.png",
      step4TestMode: "/docs/get-started/images/quickstart/step-4-test-mode.png"
    },
    quickstartCanvas: "/docs/get-started/images/quickstart-canvas.png"
  },
  knowledge: {
    addASource: "/docs/guides/knowledge/images/add-a-source.png",
    addRetrieverPicker: "/docs/guides/knowledge/images/add-retriever-picker.png",
    attachKnowledgeGambit: "/docs/guides/knowledge/images/attach-knowledge-gambit.png",
    configureARetriever: "/docs/guides/knowledge/images/configure-a-retriever.png",
    createKnowledgeBaseDialog: "/docs/guides/knowledge/images/create-knowledge-base-dialog.png",
    customRetrieverDialog: "/docs/guides/knowledge/images/custom-retriever-dialog.png",
    datasetTab: "/docs/guides/knowledge/images/dataset-tab.png",
    documentChangeReindexLifecycle: "/docs/guides/knowledge/images/document-change-reindex-lifecycle.svg",
    documentsTabPreview: "/docs/guides/knowledge/images/documents-tab-preview.png",
    importAWebsite: "/docs/guides/knowledge/images/import-a-website.png",
    kbList: "/docs/guides/knowledge/images/kb-list.png",
    retrieverIndexStates: "/docs/guides/knowledge/images/retriever-index-states.svg",
    retrieverQueryPipeline: "/docs/guides/knowledge/images/retriever-query-pipeline.svg",
    standardVsParentChildChunking: "/docs/guides/knowledge/images/standard-vs-parent-child-chunking.svg",
    syncManagement: "/docs/guides/knowledge/images/sync-management.png",
    testRetriever: "/docs/guides/knowledge/images/test-retriever.png"
  },
  liveChat: {
    agentWorkspace: "/docs/guides/live-chat/images/agent-workspace.png",
    assignedAgentPicker: "/docs/guides/live-chat/images/assigned-agent-picker.png",
    automaticAssignmentRouting: "/docs/guides/live-chat/images/automatic-assignment-routing.svg",
    availabilityDropdown: "/docs/guides/live-chat/images/availability-dropdown.png",
    cannedResponses: "/docs/guides/live-chat/images/canned-responses.png",
    composerPreferences: "/docs/guides/live-chat/images/composer-preferences.png",
    copilotPanel: "/docs/guides/live-chat/images/copilot-panel.png",
    filterConversationsDialog: "/docs/guides/live-chat/images/filter-conversations-dialog.png",
    inboxAnalyticsTab: "/docs/guides/live-chat/images/inbox-analytics-tab.png",
    inboxAvailabilityTab: "/docs/guides/live-chat/images/inbox-availability-tab.png",
    inboxesSettings: "/docs/guides/live-chat/images/inboxes-settings.png",
    inboxRoutingTab: "/docs/guides/live-chat/images/inbox-routing-tab.png",
    labelsSettings: "/docs/guides/live-chat/images/labels-settings.png",
    newAttributeDialog: "/docs/guides/live-chat/images/new-attribute-dialog.png",
    newCannedResponseDialog: "/docs/guides/live-chat/images/new-canned-response-dialog.png",
    notificationPreferences: "/docs/guides/live-chat/images/notification-preferences.png",
    notificationsPage: "/docs/guides/live-chat/images/notifications-page.png",
    pickupTargetCard: "/docs/guides/live-chat/images/pickup-target-card.png",
    resolveMenu: "/docs/guides/live-chat/images/resolve-menu.png",
    searchDialog: "/docs/guides/live-chat/images/search-dialog.png",
    seatGateConversation: "/docs/guides/live-chat/images/seat-gate-conversation.png",
    seatsSettings: "/docs/guides/live-chat/images/seats-settings.png",
    workTheInbox: "/docs/guides/live-chat/images/work-the-inbox.png"
  },
  security: {
    addOpenRouterKeyDialog: "/docs/guides/security/images/add-open-router-key-dialog.png",
    auditChainVerified: "/docs/guides/security/images/audit-chain-verified.png",
    auditEventDrawer: "/docs/guides/security/images/audit-event-drawer.png",
    auditExportDialog: "/docs/guides/security/images/audit-export-dialog.png",
    auditLogViewer: "/docs/guides/security/images/audit-log-viewer.png",
    auditRecordLifecycle: "/docs/guides/security/images/audit-record-lifecycle.svg",
    logPrivacyRedactionEnabled: "/docs/guides/security/images/log-privacy-redaction-enabled.png",
    modelProviderEmptyState: "/docs/guides/security/images/model-provider-empty-state.png",
    privacySettings: "/docs/guides/security/images/privacy-settings.png",
    siemConnectWizard: "/docs/guides/security/images/siem-connect-wizard.png",
    supportActorMasking: "/docs/guides/security/images/support-actor-masking.svg"
  },
  tools: {
    addToolModal: "/docs/guides/tools/images/add-tool-modal.png",
    addToolsPanel: "/docs/guides/tools/images/add-tools-panel.png",
    codekitActionEditor: "/docs/guides/tools/images/codekit-action-editor.png",
    codekitConnectToolkitsDialog: "/docs/guides/tools/images/codekit-connect-toolkits-dialog.png",
    codekitSchemaDialog: "/docs/guides/tools/images/codekit-schema-dialog.png",
    codekitToolkit: "/docs/guides/tools/images/codekit-toolkit.png",
    codekitToolkitAnatomy: "/docs/guides/tools/images/codekit-toolkit-anatomy.svg",
    createActionDialog: "/docs/guides/tools/images/create-action-dialog.png",
    createCodekitDialog: "/docs/guides/tools/images/create-codekit-dialog.png",
    responseFilteringDialog: "/docs/guides/tools/images/response-filtering-dialog.png",
    toolGambitEditorAgentMode: "/docs/guides/tools/images/tool-gambit-editor-agent-mode.png",
    toolGambitEditorWorkflowMode: "/docs/guides/tools/images/tool-gambit-editor-workflow-mode.png",
    toolkitDetail: "/docs/guides/tools/images/toolkit-detail.png",
    toolsConnected: "/docs/guides/tools/images/tools-connected.png",
    toolsDirectory: "/docs/guides/tools/images/tools-directory.png",
    viewActionsModal: "/docs/guides/tools/images/view-actions-modal.png",
    workflowModeResponse: "/docs/guides/tools/images/workflow-mode-response.png"
  }
};

export const urls = {
  app: "https://dashboard.hellotars.com",
  site: "https://hellotars.com",
  support: "mailto:support@hellotars.com",
  status: "https://status.hellotars.com"
};

At the end of this page, the pages you picked from a site are documents your agent can answer from. Tars fetches them again as often as you ask.

<Browser url={urls.app}>
  <img src={img.knowledge.importAWebsite} alt="Website discovery preview listing discovered pages grouped by path, each with a checkbox" />
</Browser>

## Before you start

Before you start, sign in to <a href={urls.app}>the dashboard</a> and select the organization you want to work in.

* You need a knowledge base, or you can start one from the same dialog. See [Create a knowledge base](/docs/guides/knowledge/create-a-knowledge-base).
* Your role must allow resource management.

## Start the import

<Steps>
  <Step title="Open the knowledge base and select Add more">
    The **Add a source** dialog opens. Its description reads **Fetch a website or upload a file.**
  </Step>

  <Step title="Select Import from website">
    The website tab shows the settings on the left and a preview pane on the right.
  </Step>

  <Step title="Enter the Website URL">
    Type the address of the site. Discovery starts shortly after you stop typing.
  </Step>

  <Step title="Review the discovery preview">
    The preview shows the site identity and the discovered pages as a tree.
  </Step>

  <Step title="Select the pages to import">
    Clear **Include all discovered pages** to pick pages, or uncheck whole sections.
  </Step>

  <Step title="Select Start fetching">
    The button reads **Starting…** while the import begins. The knowledge base page then shows its progress.
  </Step>
</Steps>

The same website tab appears in the create dialog, so a new knowledge base can start from an import. See [Create a knowledge base](/docs/guides/knowledge/create-a-knowledge-base).

While a retriever is building its index, **Add more** is disabled with the tooltip **Locked while a retriever is building**. See [Manage documents](/docs/guides/knowledge/manage-documents) for what the lock covers.

## The Website URL field

The **Website URL** field validates as you type. The submit button stays disabled while the field is empty or shows one of these messages:

| Message                                            | Cause                             |
| -------------------------------------------------- | --------------------------------- |
| **Enter a valid URL**                              | The text does not parse as a URL. |
| **Only http and https URLs are supported**         | The URL uses another scheme.      |
| **Enter a valid website domain, like example.com** | The host is not a valid domain.   |

If you look up too many URLs in a row, a rate-limit note appears. It asks you to wait a minute, then edit the URL to try again.

## Read the discovery preview

Discovery lists candidate pages before any content is imported. The right pane starts with **Enter a website to see its pages** and fills as discovery runs.

| Region             | What it shows                                                                                                                          |
| ------------------ | -------------------------------------------------------------------------------------------------------------------------------------- |
| Site identity card | Favicon or preview image, the site title or host, its description, and an optional brand-color dot.                                    |
| Count chip         | **Found {"{n}"} pages…** while discovering, then **{"{n}"} candidate pages found** with a **via sitemap** or **from this page** badge. |
| Estimate note      | States that the count is an estimate, that fetching may find more pages, and the page cap for this import.                             |
| Select-all row     | The **Include all discovered pages** checkbox.                                                                                         |
| Page tree          | Candidate pages grouped by path, with per-group and per-page checkboxes.                                                               |

Discovery on a large sitemap can be slow. After a few seconds the preview notes **This site has a large sitemap. Still collecting pages…**

Three special cases change the preview:

* On a site with more than 1000 discovered pages, individual page picking is unavailable. Fetch all pages, or uncheck whole sections to exclude them.
* When discovery finds no sitemap and one page at most, the preview titles itself **Pages are found as we fetch**. Start fetching, and Tars follows links from the start URL to discover the rest.
* When nothing is found, the preview says **No pages were found here.**

If discovery fails, the preview shows **Couldn't collect pages** with the reason. You can still create the knowledge base and try again from its page.

## Fetch settings

Changing a setting that affects discovery restarts the preview, so the pages you review always match the settings you submit.

| Setting          | What it does                                                                                                                             |
| ---------------- | ---------------------------------------------------------------------------------------------------------------------------------------- |
| **Max pages**    | Caps the number of fetched pages. Default 200, or the pages left on your plan when fewer.                                                |
| **Max depth**    | Caps the link hops from the start URL. Default 3.                                                                                        |
| **Media search** | Embeds the images and videos of this knowledge base so the agent can find them. Off by default. Turning it on uses extra vector storage. |

If the page cap is more than the pages left on your plan, a warning under the two fields asks you to lower the max pages value. The submit button stays disabled until the selection fits.

Open **Advanced settings** for the rest. It is collapsed by default.

| Setting                | What it does                                                                                          |
| ---------------------- | ----------------------------------------------------------------------------------------------------- |
| **Exclude paths**      | Comma-separated path globs to skip, like `/blog/*`.                                                   |
| **Render JavaScript**  | Loads pages in a browser so app-rendered content is captured. Slower.                                 |
| **Include subdomains** | Also fetches pages on subdomains of the start URL, such as docs.example.com.                          |
| **User agent**         | An optional User-Agent header for every request of this import. Up to 256 printable ASCII characters. |
| **Re-fetch schedule**  | **Never**, **Daily**, **Weekly**, or **Monthly**. Default **Never**.                                  |

Turn on **Render JavaScript** for sites whose content only appears after the page runs in a browser. With **Media search** on, JavaScript rendering is also needed to capture videos on those sites. Set **User agent** when a site blocks unknown crawlers or allows only a named one.

## Watch the import

While the fetch runs, a banner at the top of the **Documents** tab counts the scraped pages. Failed and skipped counts appear beside it when they occur. Select **Cancel** on the banner to stop the import.

The document list fills in as pages are scraped, each row with a live status badge. See [Manage documents](/docs/guides/knowledge/manage-documents) for the list during an import.

When the run ends, a toast reports the result as new, updated, and unchanged pages. Unchanged pages are detected by content hash, so a re-fetch never duplicates identical content.

## Schedule re-fetches

A schedule keeps imported pages current without manual work. The **Re-fetch schedule** options carry usage hints: **Daily** for blogs and news, **Weekly** for support centers, **Monthly** for policies and pricing. Change the schedule later from **Sync management**.

## Sync management

Select **Sync management** above the document list to open the sheet. The sheet holds the re-crawl schedules, the source settings, and the recent crawls. A knowledge base with only uploaded files has nothing on a schedule, and the sheet says so.

<Browser url={urls.app}>
  <img src={img.knowledge.syncManagement} alt="The Sync management sheet with the source picker, schedule select, and action buttons above a list of recent crawl runs" />
</Browser>

The **Re-crawl schedule** section shows one website source at a time.

| Control                  | What it does                                                                                                                                             |
| ------------------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------- |
| Source select            | Picks the website source when the knowledge base has several.                                                                                            |
| Frequency select         | Sets the schedule to **Never**, **Daily**, **Weekly**, or **Monthly**. A toast confirms the change.                                                      |
| **Sync this source now** | Re-fetches the source immediately. A toast confirms **Re-scraping this source now.** While a sync runs, the tooltip reads **A sync is already running**. |
| **Edit fetch settings**  | Opens the **Fetch settings** dialog for this source.                                                                                                     |

Both action buttons are disabled while a retriever builds, with the tooltip **Locked while a retriever is building**. A source that never imported successfully shows **Import this website successfully before changing its sync settings.**

### The Fetch settings dialog

The dialog notes **Applies on next sync for {"{source}"}**. It holds **Max pages** in the range 1 to 5000, **Max depth**, **User agent**, **Render JavaScript**, **Include subdomains**, and **Exclude paths**. Select **Save**, and a toast confirms **Source fetch settings updated.**

## Review crawl history

Below the schedule section, each crawl run of the selected source appears as a history row.

| Element        | What it shows                                                                                                                                |
| -------------- | -------------------------------------------------------------------------------------------------------------------------------------------- |
| Status badge   | **Completed**, **With errors**, **Failed**, **Cancelled**, **Finding pages**, **Awaiting review**, **Queued**, **Running**, or **Starting**. |
| Trigger badge  | **Scheduled** for a scheduled re-sync, **Manual** for an import you started.                                                                 |
| Page counts    | **{"{scraped}"} / {"{discovered}"} pages**, or just the scraped count. Link following can overshoot the discovered estimate.                 |
| Change summary | **+{"{new}"} \~{"{updated}"} ={"{unchanged}"}**, with failed and skipped counts when present.                                                |
| Duration       | How long the run took, such as **42s** or **3m**.                                                                                            |

Expand a run to see its **Failed** and **Skipped** pages grouped by reason, with the first five URLs per reason. Page-level details are kept for 14 days, after which the row notes they expired.

## Retry a failed crawl

When a fetch fails, a toast names the reason, such as a timeout or an unreachable site. Open **Sync management** and select **Sync this source now** to run the source again.

## Review a held re-fetch

<Warning>
  Confirming a held re-fetch syncs the smaller result. Cancel instead to keep the current documents.
</Warning>

A re-fetch does not update the knowledge base on its own in two cases. The run is held as **Awaiting review**, and the import banner shows a warning.

| Warning starts with | Cause                                                                                                                     |
| ------------------- | ------------------------------------------------------------------------------------------------------------------------- |
| **Site shrank**     | A scheduled re-fetch found far fewer pages than the last sync. The warning names both page counts.                        |
| **Sync paused**     | The sync found more pages than your plan has room for. The warning names the pages found and the pages left on your plan. |

Select **Review** on the banner to open the **Review pages** dialog. Deselect pages, or add exclusions under **Advanced settings**, then confirm the fetch. Select **Cancel** to keep the current documents.

The confirm button states exactly what will run: **Fetch {"{n}"} included**, **Fetch up to {"{n}"} of {"{m}"} included**, **Fetch up to {"{n}"} pages**, or **No pages to fetch** when nothing is selected.

## Verify

The **Documents** tab lists the imported pages. The line above the list shows when the site last synced and the page count.

## Related pages

* [Create a knowledge base](/docs/guides/knowledge/create-a-knowledge-base)
* [Manage documents](/docs/guides/knowledge/manage-documents)
* [Configure a retriever](/docs/guides/knowledge/configure-a-retriever)
