hive

package

v0.0.0-...-fc12ac8 Latest Latest Go to latest Published: Apr 21, 2016 License: Apache-2.0 Imports: 19 Imported by: 9

Details

Valid go.mod file
Redistributable license
Tagged version
Stable version
Learn more about best practices

Repository

github.com/eaciit/hdc

Links

Open Source Insights

Documentation ¶

Index ¶

Constants
Variables
func CheckDataType(inputModel reflect.StructField, inputVal interface{}, dateFormat string) (output string)
func DetectDataType(in string, dateFormat string) (res string)
func InspectJson(ins []string) (out []string)
func Parse(header []string, in interface{}, m interface{}, outputType string, ...) (e error)
func QueryBuilder(clause, tablename, input string, TableModel interface{}) (retVal string)
type DuplexTerm
- func (d *DuplexTerm) Close()
- func (d *DuplexTerm) Open() (e error)
- func (d *DuplexTerm) SendInput(input string) (res HiveResult, err error)
- func (d *DuplexTerm) TestConnection() (e error)
type FieldMismatch
- func (e *FieldMismatch) Error() string
type FnHiveReceive
type Hive
- func HiveConfig(server, dbName, userid, password, path string, delimiter ...string) *Hive
- func (h *Hive) CheckDataStructure(Tablename string, TableModel interface{}) (isMatch bool, err error)
- func (h *Hive) Exec(query string, fn FnHiveReceive) (e error)
- func (h *Hive) Load(TableName, dateFormat string, TableModel interface{}) (retVal string, err error)
- func (h *Hive) LoadFile(FilePath, TableName, fileType, dateFormat string, TableModel interface{}) (retVal string, err error)
- func (h *Hive) LoadFileWithWorker(FilePath, TableName, fileType string, dateFormat string, ...) (retVal string, err error)
- func (h *Hive) Populate(query string, m interface{}) (e error)
type HiveManager
- func NewHiveManager(numWorkers int) HiveManager
- func (m *HiveManager) AssignTask(task string, wg *sync.WaitGroup)
- func (m *HiveManager) DoMonitor(wg *sync.WaitGroup)
- func (m *HiveManager) EndWorker()
- func (m *HiveManager) InProgress(result int64, wg *sync.WaitGroup)
- func (m *HiveManager) Timeout(seconds int, wg *sync.WaitGroup)
type HiveResult
type HiveWorker
- func (w *HiveWorker) Work(task string, wg *sync.WaitGroup)
type UnsupportedType
- func (e *UnsupportedType) Error() string

Constants ¶

View Source

const (
	BEE_CLI_STR      = "jdbc:hive2:"
	CLOSE_SCRIPT     = "!quit"
	BEE_CLOSED       = "(closed)>"
	TEST_CONN_SCRIPT = "show tables;"
)

View Source

const (
	BEE_TEMPLATE  = "%sbeeline -u jdbc:hive2://%s/%s"
	BEE_USER      = " -n %s"
	BEE_PASSWORD  = " -p %s"
	BEE_QUERY     = " -e \"%s\""
	PACKAGENAME   = "Hive"
	CSV_FORMAT    = " --outputFormat=csv"
	TSV_FORMAT    = " --outputFormat=tsv"
	DSV_FORMAT    = " --outputFormat=dsv --delimiterForDSV=|\t"
	DSV_DELIMITER = "|\t"
	TSV           = "tsv"
	CSV           = "csv"
	JSON          = "json"
)

Variables ¶

View Source

var JsonPart string

Functions ¶

func CheckDataType ¶

func CheckDataType(inputModel reflect.StructField, inputVal interface{}, dateFormat string) (output string)

func DetectDataType ¶

func DetectDataType(in string, dateFormat string) (res string)

func InspectJson ¶

func InspectJson(ins []string) (out []string)

func Parse ¶

func Parse(header []string, in interface{}, m interface{}, outputType string, dateFormat string) (e error)

func QueryBuilder ¶

func QueryBuilder(clause, tablename, input string, TableModel interface{}) (retVal string)

Types ¶

type DuplexTerm ¶

type DuplexTerm struct {
	Writer     *bufio.Writer
	Reader     *bufio.Reader
	Cmd        *exec.Cmd
	CmdStr     string
	Stdin      io.WriteCloser
	Stdout     io.ReadCloser
	FnReceive  FnHiveReceive
	OutputType string
	DateFormat string
}

func (*DuplexTerm) Close ¶

func (d *DuplexTerm) Close()

func (*DuplexTerm) Open ¶

func (d *DuplexTerm) Open() (e error)

func (*DuplexTerm) SendInput ¶

func (d *DuplexTerm) SendInput(input string) (res HiveResult, err error)

func (*DuplexTerm) TestConnection ¶

func (d *DuplexTerm) TestConnection() (e error)

type FieldMismatch ¶

type FieldMismatch struct {
	// contains filtered or unexported fields
}

func (*FieldMismatch) Error ¶

func (e *FieldMismatch) Error() string

type FnHiveReceive ¶

type FnHiveReceive func(HiveResult) error

type Hive ¶

type Hive struct {
	BeePath    string
	Server     string
	User       string
	Password   string
	DBName     string
	Conn       *DuplexTerm
	OutputType string
	DateFormat string
}

func HiveConfig ¶

func HiveConfig(server, dbName, userid, password, path string, delimiter ...string) *Hive

func (*Hive) CheckDataStructure ¶

func (h *Hive) CheckDataStructure(Tablename string, TableModel interface{}) (isMatch bool, err error)

func (*Hive) Exec ¶

func (h *Hive) Exec(query string, fn FnHiveReceive) (e error)

func (*Hive) Load ¶

func (h *Hive) Load(TableName, dateFormat string, TableModel interface{}) (retVal string, err error)

func (*Hive) LoadFile ¶

func (h *Hive) LoadFile(FilePath, TableName, fileType, dateFormat string, TableModel interface{}) (retVal string, err error)

func (*Hive) LoadFileWithWorker ¶

func (h *Hive) LoadFileWithWorker(FilePath, TableName, fileType string, dateFormat string, TableModel interface{}, TotalWorker int) (retVal string, err error)

loading file with worker

func (*Hive) Populate ¶

func (h *Hive) Populate(query string, m interface{}) (e error)

type HiveManager ¶

type HiveManager struct {
	FreeWorkers  chan *HiveWorker
	Tasks        chan string
	Done         chan bool
	TimeProcess  chan int64
	LastProcess  int64
	TotalTimeOut int64
}

manager model

func NewHiveManager ¶

func NewHiveManager(numWorkers int) HiveManager

initiate new manager

func (*HiveManager) AssignTask ¶

func (m *HiveManager) AssignTask(task string, wg *sync.WaitGroup)

assign task to free worker

func (*HiveManager) DoMonitor ¶

func (m *HiveManager) DoMonitor(wg *sync.WaitGroup)

do monitoring worker thats free or not

func (*HiveManager) EndWorker ¶

func (m *HiveManager) EndWorker()

func (*HiveManager) InProgress ¶

func (m *HiveManager) InProgress(result int64, wg *sync.WaitGroup)

check if a task still in progress to wait it till finish

func (*HiveManager) Timeout ¶

func (m *HiveManager) Timeout(seconds int, wg *sync.WaitGroup)

set the timeout to waiting for tasks execution

type HiveResult ¶

type HiveResult struct {
	Header     []string
	Result     []string
	ResultObj  interface{}
	DateFormat string
}

type HiveWorker ¶

type HiveWorker struct {
	WorkerId    int
	TimeProcess chan int64
	FreeWorkers chan *HiveWorker
	Context     *Hive
	IsConnOpen  bool
}

worker model

func (*HiveWorker) Work ¶

func (w *HiveWorker) Work(task string, wg *sync.WaitGroup)

do a task for worker

type UnsupportedType ¶

type UnsupportedType struct {
	Type string
}

func (*UnsupportedType) Error ¶

func (e *UnsupportedType) Error() string

Source Files ¶

View all Source files

?	: This menu
/	: Search site
f or F	: Jump to
y or Y	: Canonical URL