pkg

package
v0.0.0-...-ec9e501 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: May 21, 2025 License: MIT, MIT Imports: 12 Imported by: 0

Documentation

Index

Constants

This section is empty.

Variables

View Source
var Telemetry = &TelemetryManager{Enabled: false, Endpoint: ""}

Helper: No-op singleton if not enabled

Functions

func DisableExtension

func DisableExtension(name string) error

DisableExtension removes an extension from the enabled list

func EnableExtension

func EnableExtension(name string) error

EnableExtension adds an extension to the enabled list

func GenerateProfile

func GenerateProfile(tablePath string) error

GenerateProfile creates a profile for a Delta table

func InitConfig

func InitConfig(configFile string) error

InitConfig initializes the configuration

func InstallExtension

func InstallExtension(name string) error

InstallExtension installs a new extension

func IsExtensionEnabled

func IsExtensionEnabled(name string) bool

IsExtensionEnabled checks if an extension is enabled

func RunQualityChecks

func RunQualityChecks(tablePath string) error

RunQualityChecks performs data quality checks on a Delta table

func StartServer

func StartServer(host string, port int) error

StartServer initializes and starts the API server

func UninstallExtension

func UninstallExtension(name string) error

UninstallExtension removes an installed extension

func UpdateConfig

func UpdateConfig(updates map[string]interface{}) error

UpdateConfig updates the configuration with new values

Types

type Checkpoint

type Checkpoint struct {
	Version    int64
	Timestamp  time.Time
	FileCount  int64
	FilePaths  []string
	Schema     map[string]string
	Partitions []string
}

Checkpoint represents a Delta table checkpoint

type ColumnStats

type ColumnStats struct {
	NullCount     int64
	DistinctCount int64
	MinValue      interface{}
	MaxValue      interface{}
	AvgValue      float64
}

ColumnStats represents statistics for a single column

type Config

type Config struct {
	Server     ServerConfig     `mapstructure:"server"`
	Extensions ExtensionsConfig `mapstructure:"extensions"`
	Telemetry  TelemetryConfig  `mapstructure:"telemetry"`
}

Config represents the application configuration

func GetConfig

func GetConfig() *Config

GetConfig returns the current configuration

type DeltaTable

type DeltaTable struct {
	Path            string
	Version         int64
	LastModified    time.Time
	PartitionSchema []string
	Schema          map[string]string
	Stats           *TableStats
	LogFiles        []string
	Checkpoint      *Checkpoint
}

DeltaTable represents a Delta table

type Extension

type Extension struct {
	Name        string
	Version     string
	Description string
}

Extension represents a nessi extension

func ListExtensions

func ListExtensions() ([]Extension, error)

ListExtensions returns all available extensions

type ExtensionsConfig

type ExtensionsConfig struct {
	Enabled []string `mapstructure:"enabled"`
	Path    string   `mapstructure:"path"`
}

ExtensionsConfig represents the extensions configuration

func GetExtensionsConfig

func GetExtensionsConfig() ExtensionsConfig

GetExtensionsConfig returns the extensions configuration

type ProfileStats

type ProfileStats struct {
	ColumnName    string
	DataType      string
	TotalCount    int64
	NullCount     int64
	DistinctCount int64
	MinValue      interface{}
	MaxValue      interface{}
	AvgValue      float64
	Histogram     map[string]int64
}

ProfileStats represents the statistics for a column in a Delta table

type Profiler

type Profiler struct {
	// contains filtered or unexported fields
}

Profiler handles the profiling of Delta tables

func NewProfiler

func NewProfiler(tablePath string) *Profiler

NewProfiler creates a new Profiler instance

func (*Profiler) GenerateProfile

func (p *Profiler) GenerateProfile() (*TableProfile, error)

GenerateProfile creates a complete profile for the Delta table

func (*Profiler) GetColumnProfile

func (p *Profiler) GetColumnProfile(columnName string) (*ProfileStats, error)

GetColumnProfile returns detailed statistics for a specific column

func (*Profiler) GetPartitionInfo

func (p *Profiler) GetPartitionInfo() (map[string]string, error)

GetPartitionInfo returns information about table partitions

func (*Profiler) GetTableSize

func (p *Profiler) GetTableSize() (int64, error)

GetTableSize returns the total size of the Delta table

func (*Profiler) PrintProfile

func (p *Profiler) PrintProfile(profile *TableProfile)

PrintProfile prints the table profile in a human-readable format

type Reader

type Reader struct {
	// contains filtered or unexported fields
}

Reader handles reading from Delta tables

func NewReader

func NewReader(tablePath string) (*Reader, error)

NewReader creates a new Delta table reader

func (*Reader) Close

func (r *Reader) Close() error

Close closes the reader and releases resources

func (*Reader) GetLastModified

func (r *Reader) GetLastModified() time.Time

GetLastModified returns the last modification time

func (*Reader) GetPartitionSchema

func (r *Reader) GetPartitionSchema() []string

GetPartitionSchema returns the partition schema

func (*Reader) GetPartitionStats

func (r *Reader) GetPartitionStats(partition string) (int64, error)

GetPartitionStats returns statistics for a specific partition

func (*Reader) GetSchema

func (r *Reader) GetSchema() map[string]string

GetSchema returns the table schema

func (*Reader) GetStats

func (r *Reader) GetStats() *TableStats

GetStats returns the table statistics

func (*Reader) GetTablePath

func (r *Reader) GetTablePath() string

GetTablePath returns the path of the table being read.

func (*Reader) GetVersion

func (r *Reader) GetVersion() int64

GetVersion returns the current table version

func (*Reader) ListPartitions

func (r *Reader) ListPartitions() ([]string, error)

ListPartitions returns all available partitions

func (*Reader) ReadAll

func (r *Reader) ReadAll() (io.ReadCloser, error)

ReadAll reads all data from the table

func (*Reader) ReadAllParsed

func (r *Reader) ReadAllParsed() ([]map[string]interface{}, error)

ReadAllParsed reads all data from the table and parses it into a structured format

func (*Reader) ReadPartition

func (r *Reader) ReadPartition(partition string) (io.ReadCloser, error)

ReadPartition reads data from a specific partition

type Server

type Server struct {
	Port int
	Host string
}

Server represents the API server configuration

type ServerConfig

type ServerConfig struct {
	Host string `mapstructure:"host"`
	Port int    `mapstructure:"port"`
}

ServerConfig represents the server configuration

func GetServerConfig

func GetServerConfig() ServerConfig

GetServerConfig returns the server configuration

type TableProfile

type TableProfile struct {
	TablePath     string
	TotalRows     int64
	TotalColumns  int
	LastModified  time.Time
	ColumnStats   map[string]*ProfileStats
	TableSize     int64
	PartitionInfo map[string]string
}

TableProfile represents the complete profile of a Delta table

type TableStats

type TableStats struct {
	NumFiles        int64
	NumRecords      int64
	TotalSize       int64
	PartitionCounts map[string]int64
	ColumnStats     map[string]*ColumnStats
}

TableStats represents statistics about a Delta table

type TelemetryConfig

type TelemetryConfig struct {
	Enabled  bool   `mapstructure:"enabled"`
	Endpoint string `mapstructure:"endpoint"`
}

TelemetryConfig represents telemetry settings in config Add this to your main config struct

Telemetry TelemetryConfig `mapstructure:"telemetry"`

func GetTelemetryConfig

func GetTelemetryConfig() TelemetryConfig

GetTelemetryConfig returns the telemetry configuration

type TelemetryEvent

type TelemetryEvent struct {
	Timestamp time.Time `json:"timestamp"`
	Command   string    `json:"command"`
	Args      []string  `json:"args"`
	Success   bool      `json:"success"`
	Error     string    `json:"error,omitempty"`
	Version   string    `json:"version"`
}

TelemetryEvent represents a single telemetry event Extend fields as needed E.g., add Command, Args, Duration, Error, etc.

type TelemetryManager

type TelemetryManager struct {
	Enabled  bool
	Endpoint string
}

TelemetryManager handles sending telemetry events Usage: pkg.TelemetryManager.SendEvent(...)

func NewTelemetryManager

func NewTelemetryManager(cfg TelemetryConfig) *TelemetryManager

NewTelemetryManager creates a TelemetryManager from config

func (*TelemetryManager) SendEvent

func (tm *TelemetryManager) SendEvent(event TelemetryEvent)

SendEvent sends a telemetry event if enabled

Directories

Path Synopsis
api
aws
Package aws is a generated GoMock package.
Package aws is a generated GoMock package.
gcp
aws
gcp
Package datalake provides functionality for reading and writing Delta tables
Package datalake provides functionality for reading and writing Delta tables
Package freshness provides functionality for tracking data freshness and SLA compliance.
Package freshness provides functionality for tracking data freshness and SLA compliance.
example command
Package security provides license validation and security features
Package security provides license validation and security features

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL