index

var IndexCmd = &cobra.Command{
	Use:   "index [flags]",
	Short: "Build or rebuild the CANARY token database",
	Long: `Scan the codebase for CANARY tokens and store metadata in SQLite database.

This enables advanced features like priority ordering, keyword search, and checkpoints.
The database is stored at .canary/canary.db by default.`,
	RunE: func(cmd *cobra.Command, args []string) error {
		prompt, _ := cmd.Flags().GetString("prompt")
		if prompt != "" {
			if _, err := utils.LoadPrompt(prompt); err != nil {
				return err
			}
		}
		dbPath, _ := cmd.Flags().GetString("db")
		rootPath, _ := cmd.Flags().GetString("root")

		fmt.Printf("Indexing CANARY tokens from: %s\n", rootPath)

		db, err := storage.Open(dbPath)
		if err != nil {
			return fmt.Errorf("open database: %w", err)
		}

		defer func() { _ = db.Close() }()

		// Get git info if in a repo
		var commitHash, branch string
		if gitCmd := exec.Command("git", "rev-parse", "HEAD"); gitCmd.Dir == "" {
			if output, err := gitCmd.Output(); err == nil {
				commitHash = strings.TrimSpace(string(output))
			}
		}
		if gitCmd := exec.Command("git", "rev-parse", "--abbrev-ref", "HEAD"); gitCmd.Dir == "" {
			if output, err := gitCmd.Output(); err == nil {
				branch = strings.TrimSpace(string(output))
			}
		}

		if err := db.DeleteAllTokens(); err != nil {
			return fmt.Errorf("clear token index: %w", err)
		}

		ignorePatterns, ierr := canaryscan.LoadCanaryIgnore(rootPath)
		if ierr != nil {
			fmt.Fprintf(os.Stderr, "Warning: failed to load .canaryignore: %v\n", ierr)
		}

		grepCmd := exec.Command("grep",
			"-rn",
			"--include=*.go", "--include=*.md", "--include=*.py",
			"--include=*.js", "--include=*.ts", "--include=*.java",
			"--include=*.rb", "--include=*.rs", "--include=*.c",
			"--include=*.cpp", "--include=*.h", "--include=*.sql",
			"CANARY:",
			rootPath,
		)

		output, err := grepCmd.CombinedOutput()
		if err != nil && len(output) == 0 {
			fmt.Println("No CANARY tokens found")
			return nil
		}

		indexed := 0
		lines := strings.Split(string(output), "\n")
		for _, line := range lines {
			if line == "" {
				continue
			}

			parts := strings.SplitN(line, ":", 3)
			if len(parts) < 3 {
				continue
			}

			file := parts[0]
			if isGrepMatchIgnored(rootPath, file, ignorePatterns) {
				continue
			}
			lineNum := 0

			fmt.Sscanf(parts[1], "%d", &lineNum)
			content := parts[2]

			reqID := utils.ExtractField(content, "REQ")
			feature := utils.ExtractField(content, "FEATURE")
			aspect := utils.ExtractField(content, "ASPECT")
			status := utils.ExtractField(content, "STATUS")

			if reqID == "" || feature == "" {
				continue
			}

			docPath := utils.ExtractField(content, "DOC")
			docType := utils.ExtractField(content, "DOC_TYPE")

			if docPath != "" && docType == "" {

				firstPath := strings.Split(docPath, ",")[0]
				if strings.Contains(firstPath, ":") {
					docType = strings.Split(firstPath, ":")[0]
				}
			}

			token := &storage.Token{
				ReqID:       reqID,
				Feature:     feature,
				Aspect:      aspect,
				Status:      status,
				FilePath:    file,
				LineNumber:  lineNum,
				Test:        utils.ExtractField(content, "TEST"),
				Bench:       utils.ExtractField(content, "BENCH"),
				Owner:       utils.ExtractField(content, "OWNER"),
				Phase:       utils.ExtractField(content, "PHASE"),
				Keywords:    utils.ExtractField(content, "KEYWORDS"),
				SpecStatus:  utils.ExtractField(content, "SPEC_STATUS"),
				UpdatedAt:   utils.ExtractField(content, "UPDATED"),
				CreatedAt:   utils.ExtractField(content, "CREATED"),
				StartedAt:   utils.ExtractField(content, "STARTED"),
				CompletedAt: utils.ExtractField(content, "COMPLETED"),
				CommitHash:  commitHash,
				Branch:      branch,
				DependsOn:   utils.ExtractField(content, "DEPENDS_ON"),
				Blocks:      utils.ExtractField(content, "BLOCKS"),
				RelatedTo:   utils.ExtractField(content, "RELATED_TO"),
				DocPath:     docPath,
				DocHash:     utils.ExtractField(content, "DOC_HASH"),
				DocType:     docType,
				RawToken:    content,
				IndexedAt:   time.Now().UTC().Format(time.RFC3339),
			}

			if priorityStr := utils.ExtractField(content, "PRIORITY"); priorityStr != "" {
				if p, err := strconv.Atoi(priorityStr); err == nil {
					token.Priority = p
				} else {
					token.Priority = 5
				}
			} else {
				token.Priority = 5
			}

			if token.UpdatedAt == "" {
				token.UpdatedAt = time.Now().UTC().Format("2006-01-02")
			}
			if token.SpecStatus == "" {
				token.SpecStatus = "draft"
			}

			if err := db.UpsertToken(token); err != nil {
				fmt.Fprintf(os.Stderr, "Warning: failed to store token %s/%s: %v\n", reqID, feature, err)
				continue
			}

			indexed++
		}

		reg := sources.LoadFromRoot(rootPath)
		diagRefs, derr := canaryscan.ScanDiagramRefs(rootPath, nil, reg, ignorePatterns)
		if derr != nil {
			fmt.Fprintf(os.Stderr, "Warning: diagram ref scan failed: %v\n", derr)
		}
		if derr == nil {
			refs := make([]storage.Ref, 0, len(diagRefs))
			for _, r := range diagRefs {
				refs = append(refs, storage.Ref{ReqID: r.ReqID, Kind: "diagram", FilePath: r.File, LineNumber: r.Line})
			}
			if err := db.ReplaceRefs("diagram", refs); err != nil {
				fmt.Fprintf(os.Stderr, "Warning: failed to index diagram refs: %v\n", err)
			} else if len(refs) > 0 {
				fmt.Printf("Indexed %d diagram reference(s)\n", len(refs))
			}
		}

		migrateNotes, merr := canaryscan.ScanMigrateNotes(rootPath, nil, ignorePatterns, reg)
		if merr != nil {
			fmt.Fprintf(os.Stderr, "Warning: migrate note scan failed: %v\n", merr)
		}
		if merr == nil {
			migRefs := make([]storage.Ref, 0, len(migrateNotes))
			for _, n := range migrateNotes {

				if len(n.ReqIDs) == 0 {
					migRefs = append(migRefs, storage.Ref{ReqID: "", Kind: "migrate", FilePath: n.File, LineNumber: n.Line, Context: n.Text})
					continue
				}
				for _, id := range n.ReqIDs {
					migRefs = append(migRefs, storage.Ref{ReqID: id, Kind: "migrate", FilePath: n.File, LineNumber: n.Line, Context: n.Text})
				}
			}
			if err := db.ReplaceRefs("migrate", migRefs); err != nil {
				fmt.Fprintf(os.Stderr, "Warning: failed to index migration notes: %v\n", err)
			} else if len(migrateNotes) > 0 {
				fmt.Printf("Indexed %d migration note(s)\n", len(migrateNotes))
			}
		}

		fmt.Printf("\n✅ Indexed %d CANARY tokens\n", indexed)
		fmt.Printf("Database: %s\n", dbPath)

		if commitHash != "" {
			fmt.Printf("Commit: %s\n", commitHash[:8])
		}
		if branch != "" {
			fmt.Printf("Branch: %s\n", branch)
		}

		return nil
	},
}