Documentation
¶
Index ¶
- Constants
- Variables
- type AffiliationClient
- type AuthClientProvider
- type EnrichedMessage
- type Enricher
- func (e *Enricher) EnrichMessage(rawMessage *RawMessage, now time.Time) (*EnrichedMessage, error)
- func (e *Enricher) Find(slice []string, val string) bool
- func (e *Enricher) GetEmailAddress(rawMailString string) (mail *string)
- func (e *Enricher) GetEmailDomain(email string) string
- func (e *Enricher) GetEmailUsername(email string) string
- func (e *Enricher) GetUserName(rawMailString string) (username string)
- func (e *Enricher) IsValidEmail(rawMailString string) bool
- func (e *Enricher) RemoveSpecialCharactersFromString(s string) (val *string)
- func (e *Enricher) SanitizeEmails(emails []string) string
- type Fetcher
- type GoogleGroupMessage
- type GoogleGroupMessageThread
- type GoogleGroupMessages
- type HeadersData
- type HitSource
- type Hits
- type Manager
- type NHits
- type NestedHits
- type NestedRawHits
- type RawHits
- type RawMessage
- type TopHits
Constants ¶
const ( // GoogleGroups ... GoogleGroups = "googlegroups" // Unknown ... Unknown = "Unknown" // CredentialsSSMParamName from ssm CredentialsSSMParamName = "insights_googlegroups_credentials" // TokenSSMParamName from ssm TokenSSMParamName = "insights_googlegroups_token" // MaxNumberOfMessages from gmail MaxNumberOfMessages = 10000000 // MaxConcurrentRequests ... MaxConcurrentRequests = 10000 )
Variables ¶
var ( // GoogleGroupRichMapping ... GoogleGroupRichMapping = []byte(`{"mappings":{"dynamic_templates":[{"notanalyzed":{"match":"*","match_mapping_type":"string","mapping":{"type":"keyword"}}},{"int_to_float":{"match":"*","match_mapping_type":"long","mapping":{"type":"float"}}},{"formatdate":{"match":"*","match_mapping_type":"date","mapping":{"format":"strict_date_optional_time||epoch_millis","type":"date"}}}]}}`) // GoogleGroupRawMapping ... GoogleGroupRawMapping = []byte(`{"mappings":{"dynamic":true,"properties":{"metadata__updated_on":{"type":"date"},"data":{"properties":{"body":{"dynamic":false,"properties":{}}}}}}}`) // DefaultDateTime ... DefaultDateTime = time.Date(1970, 1, 1, 0, 0, 0, 0, time.UTC) )
Functions ¶
This section is empty.
Types ¶
type AffiliationClient ¶
type AffiliationClient interface {
GetIdentityByUser(key string, value string) (*affiliation.AffIdentity, error)
AddIdentity(identity *affiliation.Identity) bool
GetOrganizations(uuid string, projectSlug string) *[]affiliation.Enrollment
}
AffiliationClient manages user identity
type AuthClientProvider ¶
AuthClientProvider interacts with auth0 server
type EnrichedMessage ¶
type EnrichedMessage struct {
From string `json:"from"`
Date time.Time `json:"date"`
To []string `json:"to"`
MessageID string `json:"message_id"`
InReplyTo string `json:"in_reply_to"`
References string `json:"references"`
Subject string `json:"subject"`
Topic string `json:"topic"`
MessageBody string `json:"message_body"`
TopicID string `json:"topic_id"`
BackendVersion string `json:"backend_version"`
UUID string `json:"uuid"`
Origin string `json:"origin"`
MetadataUpdatedOn time.Time `json:"metadata__updated_on"`
BackendName string `json:"backend_name"`
MetadataTimestamp time.Time `json:"metadata__timestamp"`
MetadataEnrichedOn time.Time `json:"metadata__enriched_on"`
ProjectSlug string `json:"project_slug"`
GroupName string `json:"group_name"`
Project string `json:"project"`
Root bool `json:"root"`
FromBot bool `json:"from_bot"`
ChangedAt time.Time `json:"changed_at"`
AuthorName string `json:"author_name"`
AuthorID string `json:"author_id"`
AuthorUUID string `json:"author_uuid"`
AuthorOrgName string `json:"author_org_name"`
AuthorUserName string `json:"author_user_name"`
AuthorBot bool `json:"author_bot"`
AuthorMultiOrgNames []string `json:"author_multi_org_names"`
MboxAuthorDomain string `json:"mbox_author_domain"`
IsGoogleGroupMessage int `json:"is_google_group_message"`
Timezone int `json:"timezone"`
ViaCommunityGroup bool `json:"via_community_group"`
}
EnrichedMessage ...
type Enricher ¶
type Enricher struct {
DSName string // Datasource will be used as key for ES
ElasticSearchProvider *elastic.ClientProvider
// contains filtered or unexported fields
}
Enricher contains google groups datasource enrich logic
func NewEnricher ¶
func NewEnricher(esClientProvider *elastic.ClientProvider, affiliationsClientProvider *affiliation.Affiliation) *Enricher
NewEnricher initiates a new Enricher
func (*Enricher) EnrichMessage ¶
func (e *Enricher) EnrichMessage(rawMessage *RawMessage, now time.Time) (*EnrichedMessage, error)
EnrichMessage enriches raw message
func (*Enricher) Find ¶
Find takes a slice and looks for an element in it. If found it will return it's true, otherwise it will return false.
func (*Enricher) GetEmailAddress ¶
GetEmailAddress ...
func (*Enricher) GetEmailDomain ¶
GetEmailDomain ...
func (*Enricher) GetEmailUsername ¶
GetEmailUsername ...
func (*Enricher) GetUserName ¶
GetUserName ...
func (*Enricher) IsValidEmail ¶
IsValidEmail validates email string
func (*Enricher) RemoveSpecialCharactersFromString ¶
RemoveSpecialCharactersFromString ...
func (*Enricher) SanitizeEmails ¶
SanitizeEmails returns the first well formed email address for a given user it filters out norely emails and emails with ellipsis
type Fetcher ¶
type Fetcher struct {
DSName string
HTTPClientProvider *http.ClientProvider
ElasticSearchProvider *elastic.ClientProvider
BackendVersion string
DateFrom time.Time
GroupName string
ProjectSlug string
Project string
}
Fetcher contains GoogleGroups datasource fetch logic
func NewFetcher ¶
func NewFetcher(groupName, projectSlug, project string, httpClientProvider *http.ClientProvider, esClientProvider *elastic.ClientProvider) *Fetcher
NewFetcher initiates a new GoogleGroups fetcher
type GoogleGroupMessage ¶
type GoogleGroupMessage struct {
ID string `json:"id"`
Author string `json:"author"`
Date string `json:"date"`
File string `json:"file"`
Message string `json:"message"`
}
GoogleGroupMessage ...
type GoogleGroupMessageThread ¶
type GoogleGroupMessageThread struct {
Topic string `json:"topic"`
ID string `json:"id"`
Messages []*GoogleGroupMessage `json:"messages"`
}
GoogleGroupMessageThread ...
type GoogleGroupMessages ¶
type GoogleGroupMessages struct {
Messages []*GoogleGroupMessageThread
}
GoogleGroupMessages ...
type HeadersData ¶
type HeadersData struct {
// Date is the date the message was originally sent
Date string
// MessageID is the message id
MessageID string
// InReplyTo is who the email was sent to. This can contain multiple
// addresses if the email was forwarded.
InReplyTo string
// References
References string
// Sender is the entity that originally created and sent the message
Sender string
// From is the name - email address combo of the email author
From string
// Subject is the subject of the email
Subject string
// To is the email recipient.
To []string
// DeliveredTo is to whom the email was sent to. This can contain multiple
// addresses if the email was forwarded.
DeliveredTo []string
MailingList string
}
HeadersData struct
type Manager ¶
type Manager struct {
Slug string
GroupName string
SHConnString string
FetcherBackendVersion string
EnricherBackendVersion string
Fetch bool
Enrich bool
ESUrl string
ESUsername string
ESPassword string
ESIndex string
FromDate *time.Time
HTTPTimeout time.Duration
Project string
FetchSize int
EnrichSize int
AffBaseURL string
ESCacheURL string
ESCacheUsername string
ESCachePassword string
AuthGrantType string
AuthClientID string
AuthClientSecret string
AuthAudience string
Auth0URL string
Environment string
WebHookURL string
// contains filtered or unexported fields
}
Manager describes google groups manager
func NewManager ¶
func NewManager(slug, groupName, shConnStr, fetcherBackendVersion, enricherBackendVersion string, fetch bool, enrich bool, eSUrl string, esUser string, esPassword string, esIndex string, fromDate *time.Time, project string, fetchSize int, enrichSize int, affBaseURL, esCacheURL, esCacheUsername, esCachePassword, authGrantType, authClientID, authClientSecret, authAudience, authURL, env string) (*Manager, error)
NewManager initiates google groups manager instance
func (*Manager) GetTotalQueuedTask ¶
GetTotalQueuedTask get total number of queued tasks
type NestedHits ¶
NestedHits is the actual hit data
type NestedRawHits ¶
type NestedRawHits struct {
ID string `json:"_id"`
Source RawMessage `json:"_source"`
}
NestedRawHits is the actual hit data
type RawMessage ¶
type RawMessage struct {
From string `json:"from"`
Date time.Time `json:"date"`
To []string `json:"to"`
MessageID string `json:"message_id"`
InReplyTo string `json:"in_reply_to"`
References string `json:"references"`
Subject string `json:"subject"`
MessageBody string `json:"message_body"`
TopicID string `json:"topic_id"`
Topic string `json:"topic"`
BackendVersion string `json:"backend_version"`
UUID string `json:"uuid"`
Origin string `json:"origin"`
MetadataUpdatedOn time.Time `json:"metadata__updated_on"`
BackendName string `json:"backend_name"`
MetadataTimestamp time.Time `json:"metadata__timestamp"`
ProjectSlug string `json:"project_slug"`
GroupName string `json:"group_name"`
Project string `json:"project"`
ChangedAt time.Time `json:"changed_at"`
Timezone int `json:"timezone"`
}
RawMessage represents GoogleGroups raw message