Documentation
¶
Index ¶
Constants ¶
This section is empty.
Variables ¶
View Source
var SourceTablesRegistry = map[string][]*SourceTable{
"adjust": {
{Name: "campaigns", PrimaryKey: "id", IncKey: "created", IncStrategy: "merge"},
{Name: "creatives", PrimaryKey: "id", IncKey: "created", IncStrategy: "merge"},
{Name: "events", PrimaryKey: "id", IncKey: "created", IncStrategy: "merge"},
{Name: "custom", PrimaryKey: "id", IncKey: "created", IncStrategy: "merge"},
},
"airtable": {
{Name: "<base_id>/<table_name>", PrimaryKey: "", IncKey: "", IncStrategy: "replace"},
},
"allium": {
{Name: "query:<query_id>", PrimaryKey: "", IncKey: "", IncStrategy: "replace"},
},
"anthropic": {
{Name: "claude_code_usage", PrimaryKey: "", IncKey: "date", IncStrategy: "merge"},
{Name: "usage_report", PrimaryKey: "", IncKey: "", IncStrategy: "replace"},
{Name: "cost_report", PrimaryKey: "", IncKey: "", IncStrategy: "replace"},
{Name: "organization", PrimaryKey: "", IncKey: "", IncStrategy: "replace"},
{Name: "workspaces", PrimaryKey: "", IncKey: "", IncStrategy: "replace"},
{Name: "api_keys", PrimaryKey: "", IncKey: "", IncStrategy: "replace"},
{Name: "invites", PrimaryKey: "", IncKey: "", IncStrategy: "replace"},
{Name: "users", PrimaryKey: "", IncKey: "", IncStrategy: "replace"},
{Name: "workspace_members", PrimaryKey: "", IncKey: "", IncStrategy: "replace"},
},
"applovin": {
{Name: "publisher-report", PrimaryKey: "day", IncKey: "day", IncStrategy: "merge"},
{Name: "advertiser-report", PrimaryKey: "day", IncKey: "day", IncStrategy: "merge"},
{Name: "advertiser-probabilistic-report", PrimaryKey: "day", IncKey: "day", IncStrategy: "merge"},
{Name: "advertiser-ska-report", PrimaryKey: "day", IncKey: "day", IncStrategy: "merge"},
},
"applovinmax": {
{Name: "user_ad_revenue", PrimaryKey: "partition_date", IncKey: "partition_date", IncStrategy: "merge"},
},
"appsflyer": {
{Name: "campaigns", PrimaryKey: "", IncKey: "", IncStrategy: "replace"},
{Name: "creatives", PrimaryKey: "", IncKey: "", IncStrategy: "replace"},
},
"appstore": {
{Name: "app-downloads-detailed", PrimaryKey: "", IncKey: "processing_date", IncStrategy: "merge"},
{Name: "app-store-discovery-and-engagement-detailed", PrimaryKey: "", IncKey: "processing_date", IncStrategy: "merge"},
{Name: "app-sessions-detailed", PrimaryKey: "", IncKey: "processing_date", IncStrategy: "merge"},
{Name: "app-store-installation-and-deletion-detailed", PrimaryKey: "", IncKey: "processing_date", IncStrategy: "merge"},
{Name: "app-store-purchases-detailed", PrimaryKey: "", IncKey: "processing_date", IncStrategy: "merge"},
{Name: "app-crashes-expanded", PrimaryKey: "", IncKey: "processing_date", IncStrategy: "merge"},
},
"asana": {
{Name: "workspaces", PrimaryKey: "", IncKey: "", IncStrategy: "replace"},
{Name: "projects", PrimaryKey: "", IncKey: "", IncStrategy: "replace"},
{Name: "sections", PrimaryKey: "", IncKey: "", IncStrategy: "replace"},
{Name: "tags", PrimaryKey: "", IncKey: "", IncStrategy: "replace"},
{Name: "tasks", PrimaryKey: "gid", IncKey: "modified_at", IncStrategy: "merge"},
{Name: "stories", PrimaryKey: "", IncKey: "", IncStrategy: "replace"},
{Name: "teams", PrimaryKey: "", IncKey: "", IncStrategy: "replace"},
{Name: "users", PrimaryKey: "", IncKey: "", IncStrategy: "replace"},
},
"attio": {
{Name: "objects", PrimaryKey: "", IncKey: "", IncStrategy: "replace"},
{Name: "records:{object_api_slug}", PrimaryKey: "", IncKey: "", IncStrategy: "replace"},
{Name: "lists", PrimaryKey: "", IncKey: "", IncStrategy: "replace"},
{Name: "list_entries:{list_id}", PrimaryKey: "", IncKey: "", IncStrategy: "replace"},
{Name: "all_list_entries:{object_api_slug}", PrimaryKey: "", IncKey: "", IncStrategy: "replace"},
},
"bruin": {
{Name: "pipelines", PrimaryKey: "", IncKey: "", IncStrategy: "replace"},
{Name: "assets", PrimaryKey: "", IncKey: "", IncStrategy: "replace"},
},
"chess": {
{Name: "profiles", PrimaryKey: "", IncKey: "", IncStrategy: "replace"},
{Name: "games", PrimaryKey: "", IncKey: "", IncStrategy: "replace"},
{Name: "archives", PrimaryKey: "", IncKey: "", IncStrategy: "replace"},
},
"posthog": {
{Name: "persons", PrimaryKey: "id", IncKey: "last_seen_at", IncStrategy: "merge"},
{Name: "feature_flags", PrimaryKey: "id", IncKey: "updated_at", IncStrategy: "merge"},
{Name: "events", PrimaryKey: "id", IncKey: "timestamp", IncStrategy: "append"},
{Name: "cohorts", PrimaryKey: "id", IncKey: "last_calculation", IncStrategy: "merge"},
{Name: "event_definitions", PrimaryKey: "id", IncKey: "last_updated_at", IncStrategy: "merge"},
{Name: "property_definitions:event", PrimaryKey: "id", IncKey: "updated_at", IncStrategy: "merge"},
{Name: "property_definitions:person", PrimaryKey: "id", IncKey: "updated_at", IncStrategy: "merge"},
{Name: "property_definitions:session", PrimaryKey: "id", IncKey: "updated_at", IncStrategy: "merge"},
{Name: "annotations", PrimaryKey: "id", IncKey: "updated_at", IncStrategy: "merge"},
},
"clickup": {
{Name: "user", PrimaryKey: "id", IncKey: "", IncStrategy: "merge"},
{Name: "teams", PrimaryKey: "id", IncKey: "", IncStrategy: "merge"},
{Name: "spaces", PrimaryKey: "id", IncKey: "", IncStrategy: "merge"},
{Name: "lists", PrimaryKey: "id", IncKey: "", IncStrategy: "merge"},
{Name: "tasks", PrimaryKey: "id", IncKey: "date_updated", IncStrategy: "merge"},
},
"couchbase": {},
"cursor": {
{Name: "team_members", PrimaryKey: "", IncKey: "", IncStrategy: "replace"},
{Name: "daily_usage_data", PrimaryKey: "", IncKey: "", IncStrategy: "replace"},
{Name: "team_spend", PrimaryKey: "", IncKey: "", IncStrategy: "replace"},
{Name: "filtered_usage_events", PrimaryKey: "", IncKey: "", IncStrategy: "replace"},
},
"customerio": {
{Name: "activities", PrimaryKey: "id", IncKey: "", IncStrategy: "replace"},
{Name: "broadcasts", PrimaryKey: "id", IncKey: "updated", IncStrategy: "merge"},
{Name: "broadcast_actions", PrimaryKey: "id", IncKey: "updated", IncStrategy: "merge"},
{Name: "broadcast_action_metrics:period", PrimaryKey: "broadcast_id, action_id, period, step_index", IncKey: "", IncStrategy: "replace"},
{Name: "broadcast_messages", PrimaryKey: "id", IncKey: "", IncStrategy: "merge"},
{Name: "broadcast_metrics:period", PrimaryKey: "broadcast_id, period, step_index", IncKey: "", IncStrategy: "replace"},
{Name: "campaigns", PrimaryKey: "id", IncKey: "updated", IncStrategy: "merge"},
{Name: "campaign_actions", PrimaryKey: "id", IncKey: "updated", IncStrategy: "merge"},
{Name: "campaign_action_metrics:period", PrimaryKey: "campaign_id, action_id, period, step_index", IncKey: "", IncStrategy: "replace"},
{Name: "campaign_messages", PrimaryKey: "id", IncKey: "", IncStrategy: "merge"},
{Name: "campaign_metrics:period", PrimaryKey: "campaign_id, period, step_index", IncKey: "", IncStrategy: "replace"},
{Name: "collections", PrimaryKey: "id", IncKey: "updated_at", IncStrategy: "merge"},
{Name: "customers", PrimaryKey: "cio_id", IncKey: "", IncStrategy: "replace"},
{Name: "customer_activities", PrimaryKey: "id", IncKey: "", IncStrategy: "replace"},
{Name: "customer_attributes", PrimaryKey: "customer_id", IncKey: "", IncStrategy: "replace"},
{Name: "customer_messages", PrimaryKey: "id", IncKey: "", IncStrategy: "merge"},
{Name: "customer_relationships", PrimaryKey: "customer_id, object_type_id, object_id", IncKey: "", IncStrategy: "replace"},
{Name: "exports", PrimaryKey: "id", IncKey: "updated_at", IncStrategy: "merge"},
{Name: "info_ip_addresses", PrimaryKey: "ip", IncKey: "", IncStrategy: "replace"},
{Name: "messages", PrimaryKey: "id", IncKey: "", IncStrategy: "merge"},
{Name: "newsletters", PrimaryKey: "id", IncKey: "updated", IncStrategy: "merge"},
{Name: "newsletter_metrics:period", PrimaryKey: "newsletter_id, period, step_index", IncKey: "", IncStrategy: "replace"},
{Name: "newsletter_test_groups", PrimaryKey: "id", IncKey: "", IncStrategy: "replace"},
{Name: "object_types", PrimaryKey: "id", IncKey: "", IncStrategy: "replace"},
{Name: "objects", PrimaryKey: "object_type_id, object_id", IncKey: "", IncStrategy: "replace"},
{Name: "reporting_webhooks", PrimaryKey: "id", IncKey: "", IncStrategy: "replace"},
{Name: "segments", PrimaryKey: "id", IncKey: "updated_at", IncStrategy: "merge"},
{Name: "sender_identities", PrimaryKey: "id", IncKey: "", IncStrategy: "replace"},
{Name: "subscription_topics", PrimaryKey: "id", IncKey: "", IncStrategy: "replace"},
{Name: "transactional_messages", PrimaryKey: "id", IncKey: "", IncStrategy: "replace"},
{Name: "workspaces", PrimaryKey: "id", IncKey: "", IncStrategy: "replace"},
},
"db2": {},
"docebo": {
{Name: "branches", PrimaryKey: "", IncKey: "", IncStrategy: "replace"},
{Name: "categories", PrimaryKey: "", IncKey: "", IncStrategy: "replace"},
{Name: "certifications", PrimaryKey: "", IncKey: "", IncStrategy: "replace"},
{Name: "course_enrollments", PrimaryKey: "", IncKey: "", IncStrategy: "replace"},
{Name: "course_fields", PrimaryKey: "", IncKey: "", IncStrategy: "replace"},
{Name: "course_learning_objects", PrimaryKey: "", IncKey: "", IncStrategy: "replace"},
{Name: "courses", PrimaryKey: "", IncKey: "", IncStrategy: "replace"},
{Name: "external_training", PrimaryKey: "", IncKey: "", IncStrategy: "replace"},
{Name: "group_members", PrimaryKey: "", IncKey: "", IncStrategy: "replace"},
{Name: "groups", PrimaryKey: "", IncKey: "", IncStrategy: "replace"},
{Name: "learning_plan_course_enrollments", PrimaryKey: "", IncKey: "", IncStrategy: "replace"},
{Name: "learning_plan_enrollments", PrimaryKey: "", IncKey: "", IncStrategy: "replace"},
{Name: "learning_plans", PrimaryKey: "", IncKey: "", IncStrategy: "replace"},
{Name: "sessions", PrimaryKey: "", IncKey: "", IncStrategy: "replace"},
{Name: "user_fields", PrimaryKey: "", IncKey: "", IncStrategy: "replace"},
{Name: "users", PrimaryKey: "", IncKey: "", IncStrategy: "replace"},
},
"dynamodb": {},
"dune": {
{Name: "queries"},
{Name: "query:<id>"},
{Name: "query:<id>:<params>"},
{Name: "sql:<raw SQL>"},
},
"elasticsearch": {},
"facebookads": {
{Name: "campaigns", PrimaryKey: "id", IncKey: "", IncStrategy: "replace"},
{Name: "ad_sets", PrimaryKey: "id", IncKey: "", IncStrategy: "replace"},
{Name: "ads", PrimaryKey: "id", IncKey: "", IncStrategy: "replace"},
{Name: "ad_creatives", PrimaryKey: "id", IncKey: "", IncStrategy: "replace"},
{Name: "leads", PrimaryKey: "id", IncKey: "", IncStrategy: "replace"},
{Name: "facebook_insights", PrimaryKey: "date_start", IncKey: "date_start", IncStrategy: "merge"},
},
"fireflies": {
{Name: "active_meetings", PrimaryKey: "", IncKey: "", IncStrategy: "replace"},
{Name: "analytics", PrimaryKey: "", IncKey: "end_time", IncStrategy: "merge"},
{Name: "channels", PrimaryKey: "", IncKey: "", IncStrategy: "replace"},
{Name: "users", PrimaryKey: "", IncKey: "", IncStrategy: "replace"},
{Name: "user_groups", PrimaryKey: "", IncKey: "", IncStrategy: "replace"},
{Name: "transcripts", PrimaryKey: "", IncKey: "date", IncStrategy: "merge"},
{Name: "bites", PrimaryKey: "", IncKey: "", IncStrategy: "replace"},
{Name: "contacts", PrimaryKey: "", IncKey: "", IncStrategy: "replace"},
},
"fluxx": {
{Name: "claim", PrimaryKey: "id", IncKey: "updated_at", IncStrategy: "merge"},
{Name: "grant_request", PrimaryKey: "id", IncKey: "updated_at", IncStrategy: "merge"},
{Name: "organization", PrimaryKey: "id", IncKey: "updated_at", IncStrategy: "merge"},
{Name: "program", PrimaryKey: "id", IncKey: "updated_at", IncStrategy: "merge"},
{Name: "request_transaction", PrimaryKey: "id", IncKey: "updated_at", IncStrategy: "merge"},
{Name: "user", PrimaryKey: "id", IncKey: "updated_at", IncStrategy: "merge"},
},
"frankfurter": {
{Name: "currencies", PrimaryKey: "", IncKey: "", IncStrategy: "replace"},
{Name: "latest", PrimaryKey: "date,currency_code,base_currency", IncKey: "", IncStrategy: "merge"},
{Name: "exchange_rates", PrimaryKey: "date,currency_code,base_currency", IncKey: "date", IncStrategy: "merge"},
},
"freshdesk": {
{Name: "agents", PrimaryKey: "id", IncKey: "updated_at", IncStrategy: "merge"},
{Name: "companies", PrimaryKey: "id", IncKey: "updated_at", IncStrategy: "merge"},
{Name: "contacts", PrimaryKey: "id", IncKey: "updated_at", IncStrategy: "merge"},
{Name: "groups", PrimaryKey: "id", IncKey: "updated_at", IncStrategy: "merge"},
{Name: "roles", PrimaryKey: "id", IncKey: "updated_at", IncStrategy: "merge"},
{Name: "tickets", PrimaryKey: "id", IncKey: "updated_at", IncStrategy: "merge"},
},
"fundraiseup": {
{Name: "donations", PrimaryKey: "", IncKey: "", IncStrategy: "replace"},
{Name: "events", PrimaryKey: "", IncKey: "", IncStrategy: "replace"},
{Name: "fundraisers", PrimaryKey: "", IncKey: "", IncStrategy: "replace"},
{Name: "recurring_plans", PrimaryKey: "", IncKey: "", IncStrategy: "replace"},
{Name: "supporters", PrimaryKey: "", IncKey: "", IncStrategy: "replace"},
},
"gcs": {},
"github": {
{Name: "issues", PrimaryKey: "", IncKey: "", IncStrategy: "replace"},
{Name: "pull_requests", PrimaryKey: "", IncKey: "", IncStrategy: "replace"},
{Name: "repo_events", PrimaryKey: "id", IncKey: "created_at", IncStrategy: "merge"},
{Name: "stargazers", PrimaryKey: "", IncKey: "", IncStrategy: "replace"},
},
"googleanalytics": {
{Name: "custom:<dimensions>:<metrics>", PrimaryKey: "", IncKey: "", IncStrategy: "replace"},
},
"google_sheets": {
{Name: "<spreadsheet_id>.<sheet_name>", PrimaryKey: "", IncKey: "", IncStrategy: "replace"},
},
"googleads": {
{Name: "account_report_daily", PrimaryKey: "", IncKey: "", IncStrategy: "merge"},
{Name: "campaign_report_daily", PrimaryKey: "", IncKey: "", IncStrategy: "merge"},
{Name: "ad_group_report_daily", PrimaryKey: "", IncKey: "", IncStrategy: "merge"},
{Name: "ad_report_daily", PrimaryKey: "", IncKey: "", IncStrategy: "merge"},
{Name: "audience_report_daily", PrimaryKey: "", IncKey: "", IncStrategy: "merge"},
{Name: "keyword_report_daily", PrimaryKey: "", IncKey: "", IncStrategy: "merge"},
{Name: "click_report_daily", PrimaryKey: "", IncKey: "", IncStrategy: "merge"},
{Name: "landing_page_report_daily", PrimaryKey: "", IncKey: "", IncStrategy: "merge"},
{Name: "search_keyword_report_daily", PrimaryKey: "", IncKey: "", IncStrategy: "merge"},
{Name: "search_term_report_daily", PrimaryKey: "", IncKey: "", IncStrategy: "merge"},
},
"gorgias": {
{Name: "customers", PrimaryKey: "", IncKey: "", IncStrategy: "replace"},
{Name: "tickets", PrimaryKey: "", IncKey: "", IncStrategy: "replace"},
},
"hostaway": {
{Name: "listings", PrimaryKey: "", IncKey: "latestActivityOn", IncStrategy: "merge"},
{Name: "listing_fee_settings", PrimaryKey: "", IncKey: "updatedOn", IncStrategy: "merge"},
{Name: "listing_pricing_settings", PrimaryKey: "", IncKey: "", IncStrategy: "replace"},
{Name: "listing_agreements", PrimaryKey: "", IncKey: "", IncStrategy: "replace"},
{Name: "listing_calendars", PrimaryKey: "", IncKey: "", IncStrategy: "replace"},
{Name: "reservations", PrimaryKey: "", IncKey: "", IncStrategy: "replace"},
{Name: "conversations", PrimaryKey: "", IncKey: "", IncStrategy: "replace"},
},
"hubspot": {
{Name: "contacts"},
{Name: "companies"},
{Name: "deals"},
{Name: "tickets"},
{Name: "products"},
{Name: "quotes"},
{Name: "calls"},
{Name: "emails"},
{Name: "feedback_submissions"},
{Name: "line_items"},
{Name: "meetings"},
{Name: "notes"},
{Name: "tasks"},
{Name: "carts"},
{Name: "discounts"},
{Name: "fees"},
{Name: "invoices"},
{Name: "commerce_payments"},
{Name: "taxes"},
{Name: "owners"},
{Name: "schemas"},
},
"indeed": {
{Name: "campaigns", PrimaryKey: "", IncKey: "", IncStrategy: "replace"},
{Name: "campaign_details", PrimaryKey: "", IncKey: "", IncStrategy: "replace"},
{Name: "campaign_budget", PrimaryKey: "", IncKey: "", IncStrategy: "replace"},
{Name: "campaign_jobs", PrimaryKey: "", IncKey: "", IncStrategy: "replace"},
{Name: "campaign_properties", PrimaryKey: "", IncKey: "", IncStrategy: "replace"},
{Name: "campaign_stats", PrimaryKey: "", IncKey: "Date", IncStrategy: "merge"},
{Name: "account", PrimaryKey: "", IncKey: "", IncStrategy: "replace"},
{Name: "traffic_stats", PrimaryKey: "", IncKey: "date", IncStrategy: "merge"},
},
"influxdb": {},
"intercom": {
{Name: "contacts", PrimaryKey: "", IncKey: "updated_at", IncStrategy: "merge"},
{Name: "companies", PrimaryKey: "", IncKey: "updated_at", IncStrategy: "merge"},
{Name: "conversations", PrimaryKey: "", IncKey: "updated_at", IncStrategy: "merge"},
{Name: "tickets", PrimaryKey: "", IncKey: "updated_at", IncStrategy: "merge"},
{Name: "articles", PrimaryKey: "", IncKey: "updated_at", IncStrategy: "merge"},
{Name: "tags", PrimaryKey: "", IncKey: "", IncStrategy: "replace"},
{Name: "segments", PrimaryKey: "", IncKey: "", IncStrategy: "replace"},
{Name: "teams", PrimaryKey: "", IncKey: "", IncStrategy: "replace"},
{Name: "admins", PrimaryKey: "", IncKey: "", IncStrategy: "replace"},
{Name: "data_attributes", PrimaryKey: "", IncKey: "", IncStrategy: "replace"},
},
"isoc_pulse": {
{Name: "dnssec_adoption", PrimaryKey: "date", IncKey: "date", IncStrategy: "merge"},
{Name: "dnssec_tld_adoption", PrimaryKey: "date", IncKey: "date", IncStrategy: "merge"},
{Name: "dnssec_validation", PrimaryKey: "date", IncKey: "date", IncStrategy: "merge"},
{Name: "https", PrimaryKey: "date", IncKey: "date", IncStrategy: "merge"},
{Name: "ipv6", PrimaryKey: "date", IncKey: "date", IncStrategy: "merge"},
},
"jira": {
{Name: "projects", PrimaryKey: "id", IncKey: "", IncStrategy: "replace"},
{Name: "issues", PrimaryKey: "id", IncKey: "fields.updated", IncStrategy: "merge"},
{Name: "users", PrimaryKey: "accountId", IncKey: "", IncStrategy: "replace"},
{Name: "issue_types", PrimaryKey: "id", IncKey: "", IncStrategy: "replace"},
{Name: "statuses", PrimaryKey: "id", IncKey: "", IncStrategy: "replace"},
{Name: "priorities", PrimaryKey: "id", IncKey: "", IncStrategy: "replace"},
{Name: "resolutions", PrimaryKey: "id", IncKey: "", IncStrategy: "replace"},
{Name: "project_versions", PrimaryKey: "id", IncKey: "", IncStrategy: "replace"},
{Name: "project_components", PrimaryKey: "id", IncKey: "", IncStrategy: "replace"},
},
"kafka": {},
"rabbitmq": {},
"kinesis": {},
"klaviyo": {
{Name: "events", PrimaryKey: "id", IncKey: "datetime", IncStrategy: "merge"},
{Name: "profiles", PrimaryKey: "id", IncKey: "updated", IncStrategy: "merge"},
{Name: "campaigns", PrimaryKey: "id", IncKey: "updated_at", IncStrategy: "merge"},
{Name: "metrics", PrimaryKey: "id", IncKey: "updated", IncStrategy: "merge"},
{Name: "tags", PrimaryKey: "id", IncKey: "", IncStrategy: "replace"},
{Name: "coupons", PrimaryKey: "id", IncKey: "", IncStrategy: "replace"},
{Name: "catalog-variants", PrimaryKey: "id", IncKey: "updated", IncStrategy: "merge"},
{Name: "catalog-categories", PrimaryKey: "id", IncKey: "updated", IncStrategy: "merge"},
{Name: "catalog-items", PrimaryKey: "id", IncKey: "updated", IncStrategy: "merge"},
{Name: "flows", PrimaryKey: "id", IncKey: "updated", IncStrategy: "merge"},
{Name: "lists", PrimaryKey: "id", IncKey: "updated", IncStrategy: "merge"},
{Name: "images", PrimaryKey: "id", IncKey: "updated_at", IncStrategy: "merge"},
{Name: "segments", PrimaryKey: "id", IncKey: "updated", IncStrategy: "merge"},
{Name: "forms", PrimaryKey: "id", IncKey: "updated_at", IncStrategy: "merge"},
{Name: "templates", PrimaryKey: "id", IncKey: "updated", IncStrategy: "merge"},
},
"linear": {
{Name: "issues", PrimaryKey: "id", IncKey: "updatedAt", IncStrategy: "merge"},
{Name: "users", PrimaryKey: "id", IncKey: "updatedAt", IncStrategy: "merge"},
{Name: "workflow_states", PrimaryKey: "id", IncKey: "updatedAt", IncStrategy: "merge"},
{Name: "cycles", PrimaryKey: "id", IncKey: "updatedAt", IncStrategy: "merge"},
{Name: "attachments", PrimaryKey: "id", IncKey: "updatedAt", IncStrategy: "merge"},
{Name: "comments", PrimaryKey: "id", IncKey: "updatedAt", IncStrategy: "merge"},
{Name: "documents", PrimaryKey: "id", IncKey: "updatedAt", IncStrategy: "merge"},
{Name: "labels", PrimaryKey: "id", IncKey: "updatedAt", IncStrategy: "merge"},
{Name: "projects", PrimaryKey: "id", IncKey: "updatedAt", IncStrategy: "merge"},
{Name: "teams", PrimaryKey: "id", IncKey: "updatedAt", IncStrategy: "merge"},
{Name: "organization", PrimaryKey: "id", IncKey: "updatedAt", IncStrategy: "merge"},
},
"linkedinads": {
{Name: "custom:<dimensions>:<metrics>", PrimaryKey: "", IncKey: "", IncStrategy: "merge"},
},
"mailchimp": {
{Name: "account", PrimaryKey: "", IncKey: "", IncStrategy: "replace"},
{Name: "audiences", PrimaryKey: "id", IncKey: "date_created", IncStrategy: "merge"},
{Name: "automations", PrimaryKey: "id", IncKey: "create_time", IncStrategy: "merge"},
{Name: "campaigns", PrimaryKey: "id", IncKey: "create_time", IncStrategy: "merge"},
{Name: "connected_sites", PrimaryKey: "id", IncKey: "updated_at", IncStrategy: "merge"},
{Name: "conversations", PrimaryKey: "id", IncKey: "last_message.timestamp", IncStrategy: "merge"},
{Name: "ecommerce_stores", PrimaryKey: "id", IncKey: "updated_at", IncStrategy: "merge"},
{Name: "facebook_ads", PrimaryKey: "id", IncKey: "updated_at", IncStrategy: "merge"},
{Name: "landing_pages", PrimaryKey: "id", IncKey: "updated_at", IncStrategy: "merge"},
{Name: "reports", PrimaryKey: "id", IncKey: "send_time", IncStrategy: "merge"},
},
"mixpanel": {
{Name: "events", PrimaryKey: "distinct_id", IncKey: "time", IncStrategy: "merge"},
{Name: "profiles", PrimaryKey: "distinct_id", IncKey: "last_seen", IncStrategy: "merge"},
},
"monday": {
{Name: "account", PrimaryKey: "", IncKey: "", IncStrategy: "replace"},
{Name: "account_roles", PrimaryKey: "", IncKey: "", IncStrategy: "replace"},
{Name: "users", PrimaryKey: "", IncKey: "", IncStrategy: "replace"},
{Name: "boards", PrimaryKey: "id", IncKey: "updated_at", IncStrategy: "merge"},
{Name: "workspaces", PrimaryKey: "", IncKey: "", IncStrategy: "replace"},
{Name: "webhooks", PrimaryKey: "", IncKey: "", IncStrategy: "replace"},
{Name: "updates", PrimaryKey: "id", IncKey: "updated_at", IncStrategy: "merge"},
{Name: "teams", PrimaryKey: "", IncKey: "", IncStrategy: "replace"},
{Name: "tags", PrimaryKey: "", IncKey: "", IncStrategy: "replace"},
},
"mongo": {},
"mysql": {},
"notion": {
{Name: "<database_id>", PrimaryKey: "", IncKey: "", IncStrategy: "replace"},
},
"personio": {
{Name: "employees", PrimaryKey: "id", IncKey: "last_modified_at", IncStrategy: "merge"},
{Name: "absence_types", PrimaryKey: "id", IncKey: "", IncStrategy: "replace"},
{Name: "absences", PrimaryKey: "id", IncKey: "updated_at", IncStrategy: "merge"},
{Name: "attendances", PrimaryKey: "id", IncKey: "updated_at", IncStrategy: "merge"},
{Name: "projects", PrimaryKey: "id", IncKey: "", IncStrategy: "replace"},
{Name: "document_categories", PrimaryKey: "id", IncKey: "", IncStrategy: "replace"},
{Name: "custom_reports_list", PrimaryKey: "id", IncKey: "", IncStrategy: "replace"},
{Name: "employees_absences_balance", PrimaryKey: "employee_id,id", IncKey: "", IncStrategy: "merge"},
},
"phantombuster": {
{Name: "completed_phantoms:<agent_id>", PrimaryKey: "container_id", IncKey: "ended_at", IncStrategy: "merge"},
},
"pinterest": {
{Name: "pins", PrimaryKey: "id", IncKey: "created_at", IncStrategy: "merge"},
{Name: "boards", PrimaryKey: "id", IncKey: "created_at", IncStrategy: "merge"},
},
"pipedrive": {
{Name: "activities", PrimaryKey: "id", IncKey: "update_time", IncStrategy: "merge"},
{Name: "deals", PrimaryKey: "id", IncKey: "update_time", IncStrategy: "merge"},
{Name: "persons", PrimaryKey: "id", IncKey: "update_time", IncStrategy: "merge"},
{Name: "organizations", PrimaryKey: "id", IncKey: "update_time", IncStrategy: "merge"},
{Name: "products", PrimaryKey: "id", IncKey: "update_time", IncStrategy: "merge"},
{Name: "users", PrimaryKey: "id", IncKey: "update_time", IncStrategy: "merge"},
},
"plusvibeai": {
{Name: "campaigns", PrimaryKey: "", IncKey: "modified_at", IncStrategy: "merge"},
{Name: "leads", PrimaryKey: "", IncKey: "modified_at", IncStrategy: "merge"},
{Name: "email_accounts", PrimaryKey: "", IncKey: "timestamp_updated", IncStrategy: "merge"},
{Name: "emails", PrimaryKey: "", IncKey: "timestamp_created", IncStrategy: "merge"},
{Name: "blocklist", PrimaryKey: "", IncKey: "created_at", IncStrategy: "merge"},
{Name: "webhooks", PrimaryKey: "", IncKey: "modified_at", IncStrategy: "merge"},
{Name: "tags", PrimaryKey: "", IncKey: "modified_at", IncStrategy: "merge"},
},
"primer": {
{Name: "payments", PrimaryKey: "id", IncKey: "dateUpdated", IncStrategy: "merge"},
},
"quickbooks": {
{Name: "customers", PrimaryKey: "id", IncKey: "lastupdatedtime", IncStrategy: "merge"},
{Name: "invoices", PrimaryKey: "id", IncKey: "lastupdatedtime", IncStrategy: "merge"},
{Name: "accounts", PrimaryKey: "id", IncKey: "lastupdatedtime", IncStrategy: "merge"},
{Name: "vendors", PrimaryKey: "id", IncKey: "lastupdatedtime", IncStrategy: "merge"},
{Name: "payments", PrimaryKey: "id", IncKey: "lastupdatedtime", IncStrategy: "merge"},
},
"revenuecat": {
{Name: "projects", PrimaryKey: "id", IncKey: "", IncStrategy: "merge"},
{Name: "customers", PrimaryKey: "id", IncKey: "", IncStrategy: "merge"},
{Name: "products", PrimaryKey: "id", IncKey: "", IncStrategy: "merge"},
{Name: "entitlements", PrimaryKey: "id", IncKey: "", IncStrategy: "merge"},
{Name: "offerings", PrimaryKey: "id", IncKey: "", IncStrategy: "merge"},
},
"s3": {},
"salesforce": {
{Name: "user", PrimaryKey: "", IncKey: "", IncStrategy: "replace"},
{Name: "user_role", PrimaryKey: "", IncKey: "", IncStrategy: "replace"},
{Name: "opportunity", PrimaryKey: "id", IncKey: "last_timestamp", IncStrategy: "merge"},
{Name: "opportunity_line_item", PrimaryKey: "id", IncKey: "last_timestamp", IncStrategy: "merge"},
{Name: "opportunity_contact_role", PrimaryKey: "id", IncKey: "last_timestamp", IncStrategy: "merge"},
{Name: "account", PrimaryKey: "id", IncKey: "last_timestamp", IncStrategy: "merge"},
{Name: "contact", PrimaryKey: "id", IncKey: "", IncStrategy: "replace"},
{Name: "lead", PrimaryKey: "id", IncKey: "", IncStrategy: "replace"},
{Name: "campaign", PrimaryKey: "id", IncKey: "", IncStrategy: "replace"},
{Name: "campaign_member", PrimaryKey: "id", IncKey: "last_timestamp", IncStrategy: "merge"},
{Name: "product", PrimaryKey: "id", IncKey: "", IncStrategy: "replace"},
{Name: "pricebook", PrimaryKey: "id", IncKey: "", IncStrategy: "replace"},
{Name: "pricebook_entry", PrimaryKey: "id", IncKey: "", IncStrategy: "replace"},
{Name: "task", PrimaryKey: "id", IncKey: "last_timestamp", IncStrategy: "merge"},
{Name: "event", PrimaryKey: "id", IncKey: "last_timestamp", IncStrategy: "merge"},
},
"hana": {},
"sftp": {},
"shopify": {
{Name: "orders", PrimaryKey: "id", IncKey: "updated_at", IncStrategy: "merge"},
{Name: "customers", PrimaryKey: "id", IncKey: "updated_at", IncStrategy: "merge"},
{Name: "discounts", PrimaryKey: "id", IncKey: "updated_at", IncStrategy: "merge"},
{Name: "products", PrimaryKey: "id", IncKey: "updated_at", IncStrategy: "merge"},
{Name: "inventory_items", PrimaryKey: "id", IncKey: "updated_at", IncStrategy: "merge"},
{Name: "transactions", PrimaryKey: "id", IncKey: "id", IncStrategy: "merge"},
{Name: "balance", PrimaryKey: "currency", IncKey: "", IncStrategy: "merge"},
{Name: "events", PrimaryKey: "id", IncKey: "created_at", IncStrategy: "merge"},
{Name: "price_rules", PrimaryKey: "id", IncKey: "updated_at", IncStrategy: "merge"},
},
"slack": {
{Name: "channels", PrimaryKey: "id", IncKey: "", IncStrategy: "replace"},
{Name: "users", PrimaryKey: "id", IncKey: "", IncStrategy: "replace"},
{Name: "messages:<channel1>,<channel2>", PrimaryKey: "ts", IncKey: "ts", IncStrategy: "merge"},
{Name: "access_logs", PrimaryKey: "user_id", IncKey: "", IncStrategy: "append"},
},
"smartsheet": {
{Name: "<sheet_id>", PrimaryKey: "", IncKey: "", IncStrategy: "replace"},
},
"snapchatads": {
{Name: "organizations", PrimaryKey: "id", IncKey: "updated_at", IncStrategy: "merge"},
{Name: "fundingsources", PrimaryKey: "id", IncKey: "updated_at", IncStrategy: "merge"},
{Name: "billingcenters", PrimaryKey: "id", IncKey: "updated_at", IncStrategy: "merge"},
{Name: "adaccounts", PrimaryKey: "id", IncKey: "updated_at", IncStrategy: "merge"},
{Name: "campaigns", PrimaryKey: "id", IncKey: "updated_at", IncStrategy: "merge"},
{Name: "adsquads", PrimaryKey: "id", IncKey: "updated_at", IncStrategy: "merge"},
{Name: "ads", PrimaryKey: "id", IncKey: "updated_at", IncStrategy: "merge"},
{Name: "creatives", PrimaryKey: "id", IncKey: "updated_at", IncStrategy: "merge"},
{Name: "segments", PrimaryKey: "id", IncKey: "updated_at", IncStrategy: "merge"},
},
"socrata": {
{Name: "<dataset_id>", PrimaryKey: ":id", IncKey: "", IncStrategy: "replace"},
},
"solidgate": {
{Name: "subscriptions", PrimaryKey: "id", IncKey: "updated_at", IncStrategy: "merge"},
{Name: "apm_orders", PrimaryKey: "order_id", IncKey: "updated_at", IncStrategy: "merge"},
{Name: "card_orders", PrimaryKey: "order_id", IncKey: "updated_at", IncStrategy: "merge"},
{Name: "financial_entries", PrimaryKey: "id", IncKey: "created_at", IncStrategy: "merge"},
},
"spanner": {},
"sqlite": {},
"stripe": {
{Name: "account", PrimaryKey: "id", IncKey: "created", IncStrategy: "merge"},
{Name: "apple_pay_domain", PrimaryKey: "id", IncKey: "created", IncStrategy: "merge"},
{Name: "application_fee", PrimaryKey: "id", IncKey: "created", IncStrategy: "merge"},
{Name: "balance_transaction", PrimaryKey: "id", IncKey: "created", IncStrategy: "merge"},
{Name: "charge", PrimaryKey: "id", IncKey: "created", IncStrategy: "merge"},
{Name: "checkout_session", PrimaryKey: "id", IncKey: "created", IncStrategy: "merge"},
{Name: "coupon", PrimaryKey: "id", IncKey: "created", IncStrategy: "merge"},
{Name: "credit_note", PrimaryKey: "id", IncKey: "created", IncStrategy: "merge"},
{Name: "customer", PrimaryKey: "id", IncKey: "created", IncStrategy: "merge"},
{Name: "dispute", PrimaryKey: "id", IncKey: "created", IncStrategy: "merge"},
{Name: "event", PrimaryKey: "id", IncKey: "created", IncStrategy: "merge"},
{Name: "invoice", PrimaryKey: "id", IncKey: "created", IncStrategy: "merge"},
{Name: "invoice_item", PrimaryKey: "id", IncKey: "created", IncStrategy: "merge"},
{Name: "invoice_line_item", PrimaryKey: "id", IncKey: "created", IncStrategy: "merge"},
{Name: "payment_intent", PrimaryKey: "id", IncKey: "created", IncStrategy: "merge"},
{Name: "payment_link", PrimaryKey: "id", IncKey: "created", IncStrategy: "merge"},
{Name: "payment_method", PrimaryKey: "id", IncKey: "created", IncStrategy: "merge"},
{Name: "payment_method_domain", PrimaryKey: "id", IncKey: "created", IncStrategy: "merge"},
{Name: "payout", PrimaryKey: "id", IncKey: "created", IncStrategy: "merge"},
{Name: "plan", PrimaryKey: "id", IncKey: "created", IncStrategy: "merge"},
{Name: "price", PrimaryKey: "id", IncKey: "created", IncStrategy: "merge"},
{Name: "product", PrimaryKey: "id", IncKey: "created", IncStrategy: "merge"},
{Name: "promotion_code", PrimaryKey: "id", IncKey: "created", IncStrategy: "merge"},
{Name: "quote", PrimaryKey: "id", IncKey: "created", IncStrategy: "merge"},
{Name: "refund", PrimaryKey: "id", IncKey: "created", IncStrategy: "merge"},
{Name: "review", PrimaryKey: "id", IncKey: "created", IncStrategy: "merge"},
{Name: "setup_attempt", PrimaryKey: "id", IncKey: "created", IncStrategy: "merge"},
{Name: "setup_intent", PrimaryKey: "id", IncKey: "created", IncStrategy: "merge"},
{Name: "shipping_rate", PrimaryKey: "id", IncKey: "created", IncStrategy: "merge"},
{Name: "subscription", PrimaryKey: "id", IncKey: "created", IncStrategy: "merge"},
{Name: "subscription_item", PrimaryKey: "id", IncKey: "created", IncStrategy: "merge"},
{Name: "subscription_schedule", PrimaryKey: "id", IncKey: "created", IncStrategy: "merge"},
{Name: "tax_code", PrimaryKey: "id", IncKey: "created", IncStrategy: "merge"},
{Name: "tax_id", PrimaryKey: "id", IncKey: "created", IncStrategy: "merge"},
{Name: "tax_rate", PrimaryKey: "id", IncKey: "created", IncStrategy: "merge"},
{Name: "top_up", PrimaryKey: "id", IncKey: "created", IncStrategy: "merge"},
{Name: "transfer", PrimaryKey: "id", IncKey: "created", IncStrategy: "merge"},
{Name: "webhook_endpoint", PrimaryKey: "id", IncKey: "created", IncStrategy: "merge"},
},
"tiktokads": {
{Name: "custom:<dimensions>:<metrics>", PrimaryKey: "", IncKey: "", IncStrategy: "merge"},
},
"trustpilot": {
{Name: "reviews", PrimaryKey: "", IncKey: "", IncStrategy: "replace"},
},
"wise": {
{Name: "profiles", PrimaryKey: "id", IncKey: "", IncStrategy: "merge"},
{Name: "transfers", PrimaryKey: "id", IncKey: "created", IncStrategy: "merge"},
{Name: "balances", PrimaryKey: "id", IncKey: "modificationTime", IncStrategy: "merge"},
},
"zendesk": {
{Name: "tickets", PrimaryKey: "id", IncKey: "updated_at", IncStrategy: "merge"},
{Name: "ticket_metrics", PrimaryKey: "", IncKey: "", IncStrategy: "replace"},
{Name: "ticket_metric_events", PrimaryKey: "id", IncKey: "time", IncStrategy: "append"},
{Name: "ticket_forms", PrimaryKey: "", IncKey: "", IncStrategy: "replace"},
{Name: "users", PrimaryKey: "", IncKey: "", IncStrategy: "replace"},
{Name: "groups", PrimaryKey: "", IncKey: "", IncStrategy: "replace"},
{Name: "organizations", PrimaryKey: "", IncKey: "", IncStrategy: "replace"},
{Name: "brands", PrimaryKey: "", IncKey: "", IncStrategy: "replace"},
{Name: "sla_policies", PrimaryKey: "", IncKey: "", IncStrategy: "replace"},
{Name: "activities", PrimaryKey: "", IncKey: "", IncStrategy: "replace"},
{Name: "automations", PrimaryKey: "", IncKey: "", IncStrategy: "replace"},
{Name: "targets", PrimaryKey: "", IncKey: "", IncStrategy: "replace"},
{Name: "calls", PrimaryKey: "id", IncKey: "updated_at", IncStrategy: "merge"},
{Name: "chats", PrimaryKey: "id", IncKey: "update_timestamp", IncStrategy: "merge"},
},
"zoom": {
{Name: "meetings", PrimaryKey: "id", IncKey: "start_time", IncStrategy: "merge"},
{Name: "users", PrimaryKey: "id", IncKey: "", IncStrategy: "merge"},
{Name: "participants", PrimaryKey: "id", IncKey: "join_time", IncStrategy: "merge"},
},
}
Functions ¶
This section is empty.
Types ¶
type BasicOperator ¶
type BasicOperator struct {
// contains filtered or unexported fields
}
func NewBasicOperator ¶
func NewBasicOperator(conn config.ConnectionGetter, j jinja.RendererInterface) (*BasicOperator, error)
func (*BasicOperator) Run ¶
func (o *BasicOperator) Run(ctx context.Context, ti scheduler.TaskInstance) error
type IngestrCheckOperator ¶
type IngestrCheckOperator struct {
// contains filtered or unexported fields
}
func NewColumnCheckOperator ¶
func NewColumnCheckOperator(configs *map[pipeline.AssetType]executor.Config) *IngestrCheckOperator
func (IngestrCheckOperator) Run ¶
func (i IngestrCheckOperator) Run(ctx context.Context, ti scheduler.TaskInstance) error
type IngestrCustomCheckOperator ¶
type IngestrCustomCheckOperator struct {
// contains filtered or unexported fields
}
func NewCustomCheckOperator ¶
func NewCustomCheckOperator(configs *map[pipeline.AssetType]executor.Config) *IngestrCustomCheckOperator
func (IngestrCustomCheckOperator) Run ¶
func (i IngestrCustomCheckOperator) Run(ctx context.Context, ti scheduler.TaskInstance) error
type SeedOperator ¶ added in v0.11.128
type SeedOperator struct {
// contains filtered or unexported fields
}
func NewSeedOperator ¶ added in v0.11.128
func NewSeedOperator(conn config.ConnectionGetter, j jinja.RendererInterface) (*SeedOperator, error)
func (*SeedOperator) Run ¶ added in v0.11.128
func (o *SeedOperator) Run(ctx context.Context, ti scheduler.TaskInstance) error
type Source ¶ added in v0.11.421
type Source struct {
Name string `json:"name"`
Tables []*SourceTable `json:"tables"`
}
Source represents an ingestr source with its available tables.
func GetAllSources ¶ added in v0.11.421
func GetAllSources() []*Source
GetAllSources returns all available ingestr sources and their tables. Sources are sorted by name for consistent output.
func GetSourceTables ¶ added in v0.11.421
GetSourceTables returns the available tables for a specific ingestr source.
type SourceTable ¶ added in v0.11.421
type SourceTable struct {
Name string `json:"name"`
PrimaryKey string `json:"primary_key,omitempty"`
IncKey string `json:"incremental_key,omitempty"`
IncStrategy string `json:"incremental_strategy,omitempty"`
}
SourceTable represents a table available from an ingestr source with its metadata.
Click to show internal directories.
Click to hide internal directories.