From 65880a97c14620c535d0b04fe10708a7634bbb35 Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Wed, 9 Mar 2022 11:23:57 -0800 Subject: [PATCH 01/97] Make a proof of concept Signed-off-by: Kevin Zhang --- go/internal/feast/featurestore.go | 2 ++ go/internal/feast/featurestore_test.go | 14 ++++++++++++++ 2 files changed, 16 insertions(+) diff --git a/go/internal/feast/featurestore.go b/go/internal/feast/featurestore.go index 8eacc4a9434..3b866f143dc 100644 --- a/go/internal/feast/featurestore.go +++ b/go/internal/feast/featurestore.go @@ -3,6 +3,7 @@ package feast import ( "context" "errors" + "github.com/apache/arrow/go/v8/arrow/memory" "github.com/feast-dev/feast/go/internal/feast/model" "github.com/feast-dev/feast/go/internal/feast/onlineserving" @@ -135,6 +136,7 @@ func (fs *FeatureStore) GetOnlineFeatures( for _, groupRef := range groupedRefs { featureData, err := fs.readFromOnlineStore(ctx, groupRef.EntityKeys, groupRef.FeatureViewNames, groupRef.FeatureNames) + if err != nil { return nil, err } diff --git a/go/internal/feast/featurestore_test.go b/go/internal/feast/featurestore_test.go index 00babcc3854..80b223f65e0 100644 --- a/go/internal/feast/featurestore_test.go +++ b/go/internal/feast/featurestore_test.go @@ -2,13 +2,22 @@ package feast import ( "context" +<<<<<<< HEAD "github.com/feast-dev/feast/go/internal/feast/onlinestore" "github.com/feast-dev/feast/go/internal/feast/registry" "github.com/feast-dev/feast/go/protos/feast/types" "github.com/stretchr/testify/assert" +======= +>>>>>>> f7354334 (Make a proof of concept) "path/filepath" "runtime" "testing" + + "github.com/feast-dev/feast/go/protos/feast/types" + "github.com/stretchr/testify/assert" + + "github.com/feast-dev/feast/go/protos/feast/types" + "github.com/stretchr/testify/assert" ) // Return absolute path to the test_repo registry regardless of the working directory @@ -40,8 +49,13 @@ func TestNewFeatureStore(t *testing.T) { } func TestGetOnlineFeaturesRedis(t *testing.T) { +<<<<<<< HEAD t.Skip("@todo(achals): feature_repo isn't checked in yet") config := registry.RepoConfig{ +======= + //t.Skip("@todo(achals): feature_repo isn't checked in yet") + config := RepoConfig{ +>>>>>>> f7354334 (Make a proof of concept) Project: "feature_repo", Registry: getRegistryPath(), Provider: "local", From b32b157cf223dcfcdb424ca0f58b713b4a0b3a30 Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Fri, 11 Mar 2022 12:43:10 -0800 Subject: [PATCH 02/97] Update Signed-off-by: Kevin Zhang --- go/cmd/gologging/logging_service.go | 21 +++++++++++ go/cmd/gologging/main.go | 58 +++++++++++++++++++++++++++++ go/cmd/server/main.go | 11 ++++-- 3 files changed, 87 insertions(+), 3 deletions(-) create mode 100644 go/cmd/gologging/logging_service.go create mode 100644 go/cmd/gologging/main.go diff --git a/go/cmd/gologging/logging_service.go b/go/cmd/gologging/logging_service.go new file mode 100644 index 00000000000..1bfaa02cbaf --- /dev/null +++ b/go/cmd/gologging/logging_service.go @@ -0,0 +1,21 @@ +package logging + +import ( + "context" + + "github.com/feast-dev/feast/go/protos/feast/serving" +) + +type loggingServiceServer struct { + channel *chan string + serving.UnimplementedServingServiceServer +} + +func newLoggingServiceServer(channel *chan string) *loggingServiceServer { + return &loggingServiceServer{channel: channel} +} + +func (s *loggingServiceServer) sendLogData(ctx context.Context, log *serving.Log) (*serving.LoggingResponse, error) { + // Actually use the channel and update the channel with the new log. + // return the status +} diff --git a/go/cmd/gologging/main.go b/go/cmd/gologging/main.go new file mode 100644 index 00000000000..2c0cc0884c9 --- /dev/null +++ b/go/cmd/gologging/main.go @@ -0,0 +1,58 @@ +package logging + +import ( + "log" + "net" + + "github.com/feast-dev/feast/go/internal/feast" + "github.com/feast-dev/feast/go/protos/feast/serving" + "google.golang.org/grpc" +) + +const ( + flagFeastRepoPath = "FEAST_REPO_PATH" + flagFeastRepoConfig = "FEAST_REPO_CONFIG" + feastServerVersion = "0.18.0" +) + +type FeastEnvConfig struct { + RepoPath string `envconfig:"FEAST_REPO_PATH"` + RepoConfig string `envconfig:"FEAST_REPO_CONFIG"` + SockFile string `envconfig:"FEAST_GRPC_SOCK_FILE"` +} + +type servingServiceServer struct { + fs *feast.FeatureStore + serving.UnimplementedServingServiceServer +} + +func main() { + log_buffer := make(chan string, 1000) + startGrpcServer(log_buffer, "9000") +} + +func startGrpcServer(channel chan string, port string) { + server := newLoggingServiceServer(&channel) + // make new log service + log.Printf("Starting a gRPC server listening on %s\n", port) + lis, err := net.Listen("tcp", port) + if err != nil { + log.Fatalln(err) + } + grpcServer := grpc.NewServer() + defer grpcServer.Stop() + serving.RegisterLoggingServiceServer(grpcServer, server) + // register the servicing service with the grpc server + err = grpcServer.Serve(lis) + if err != nil { + log.Fatalln(err) + } +} + +func processLogs() { + // start a periodic flush + for { + // select function that either receives from channel or starts a subprocess to flush + // probably need to wait for that + } +} diff --git a/go/cmd/server/main.go b/go/cmd/server/main.go index 9dc6a5b966b..1c349bb1a34 100644 --- a/go/cmd/server/main.go +++ b/go/cmd/server/main.go @@ -2,13 +2,18 @@ package main import ( "fmt" + "log" + "net" + "os" + "github.com/feast-dev/feast/go/internal/feast" "github.com/feast-dev/feast/go/internal/feast/registry" "github.com/feast-dev/feast/go/protos/feast/serving" "google.golang.org/grpc" - "log" - "net" - "os" + + "github.com/feast-dev/feast/go/internal/feast" + "github.com/feast-dev/feast/go/protos/feast/serving" + "google.golang.org/grpc" ) const ( From 1996fd958830b23fb04ecd8f2352c361295b0688 Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Thu, 17 Mar 2022 13:21:55 -0700 Subject: [PATCH 03/97] revert feature store Signed-off-by: Kevin Zhang --- go/cmd/server/main.go | 54 +++++++++++++++++++++++++++----- go/cmd/server/server.go | 68 +++++++++++++++++++++++++++++++++++++++-- 2 files changed, 111 insertions(+), 11 deletions(-) diff --git a/go/cmd/server/main.go b/go/cmd/server/main.go index 1c349bb1a34..0b5aae456ae 100644 --- a/go/cmd/server/main.go +++ b/go/cmd/server/main.go @@ -4,7 +4,7 @@ import ( "fmt" "log" "net" - "os" + "time" "github.com/feast-dev/feast/go/internal/feast" "github.com/feast-dev/feast/go/internal/feast/registry" @@ -13,6 +13,7 @@ import ( "github.com/feast-dev/feast/go/internal/feast" "github.com/feast-dev/feast/go/protos/feast/serving" + "github.com/kelseyhightower/envconfig" "google.golang.org/grpc" ) @@ -23,12 +24,32 @@ const ( feastServerVersion = "0.18.0" ) +type FeastEnvConfig struct { + RepoPath string `envconfig:"FEAST_REPO_PATH"` + RepoConfig string `envconfig:"FEAST_REPO_CONFIG"` + SockFile string `envconfig:"FEAST_GRPC_SOCK_FILE"` +} + +type MemoryBuffer struct { + logs []Log +} + // TODO: Add a proper logging library such as https://github.com/Sirupsen/logrus func main() { - repoPath := os.Getenv(flagFeastRepoPath) - repoConfigJSON := os.Getenv(flagFeastRepoConfig) - sockFile := os.Getenv(flagFeastSockFile) - if repoPath == "" && repoConfigJSON == "" { + // TODO(kevjumba) Figure out how large this log channel should be. + logChannel := make(chan Log, 1000) + defer close(logChannel) + var logBuffer = MemoryBuffer{ + logs: make([]Log, 0), + } + + var feastEnvConfig FeastEnvConfig + var err error + err = envconfig.Process("feast", &feastEnvConfig) + if err != nil { + log.Fatal(err) + } + if feastEnvConfig.RepoPath == "" && feastEnvConfig.RepoConfig == "" { log.Fatalln(fmt.Sprintf("One of %s of %s environment variables must be set", flagFeastRepoPath, flagFeastRepoConfig)) } @@ -52,11 +73,12 @@ func main() { log.Fatalln(err) } defer fs.DestructOnlineStore() - startGrpcServer(fs, sockFile) + startGrpcServer(fs, logChannel, &logBuffer, feastEnvConfig.SockFile) } -func startGrpcServer(fs *feast.FeatureStore, sockFile string) { - server := newServingServiceServer(fs) +func startGrpcServer(fs *feast.FeatureStore, logChannel chan Log, logBuffer *MemoryBuffer, sockFile string) { + go processLogs(logChannel, logBuffer) + server := newServingServiceServer(fs, logChannel) log.Printf("Starting a gRPC server listening on %s\n", sockFile) lis, err := net.Listen("unix", sockFile) if err != nil { @@ -70,3 +92,19 @@ func startGrpcServer(fs *feast.FeatureStore, sockFile string) { log.Fatalln(err) } } + +func processLogs(log_channel chan Log, logBuffer *MemoryBuffer) { + // start a periodic flush + ticker := time.NewTicker(100 * time.Millisecond) + defer ticker.Stop() + for { + select { + case t := <-ticker.C: + log.Printf("time is %d", t) + log.Printf("Flushing buffer to offline storage with channel length: %d\n", len(logBuffer.logs)) + case new_log := <-log_channel: + log.Printf("Pushing %s to memory.\n", new_log.featureValues) + logBuffer.logs = append(logBuffer.logs, new_log) + } + } +} diff --git a/go/cmd/server/server.go b/go/cmd/server/server.go index 0d8c4362a2b..520e8d72be2 100644 --- a/go/cmd/server/server.go +++ b/go/cmd/server/server.go @@ -9,12 +9,28 @@ import ( ) type servingServiceServer struct { - fs *feast.FeatureStore + fs *feast.FeatureStore + logChannel chan Log serving.UnimplementedServingServiceServer } -func newServingServiceServer(fs *feast.FeatureStore) *servingServiceServer { - return &servingServiceServer{fs: fs} +type Log struct { + // Example: driver_id, customer_id + entityNames []string + // Example: val{int64_val: 5017}, val{int64_val: 1003} + entityValues []*prototypes.Value + + // Feature names is 1:1 correspondence with featureValue, featureStatus, and timestamp + featureNames []string + + featureValues []*prototypes.Value + featureStatuses []serving.FieldStatus + eventTimestamps []*timestamp.Timestamp + RequestContext map[string]*prototypes.RepeatedValue +} + +func newServingServiceServer(fs *feast.FeatureStore, logChannel chan Log) *servingServiceServer { + return &servingServiceServer{fs: fs, logChannel: logChannel} } func (s *servingServiceServer) GetFeastServingInfo(ctx context.Context, request *serving.GetFeastServingInfoRequest) (*serving.GetFeastServingInfoResponse, error) { @@ -67,3 +83,49 @@ func (s *servingServiceServer) GetOnlineFeatures(ctx context.Context, request *s return resp, nil } + +func flushToChannel(s *servingServiceServer, onlineResponse *serving.GetOnlineFeaturesResponse, request *serving.GetOnlineFeaturesRequest) { + for _, featureVector := range onlineResponse.Results { + featureValues := make([]*prototypes.Value, len(featureVector.Values)) + eventTimestamps := make([]*timestamp.Timestamp, len(featureVector.EventTimestamps)) + for idx, featureValue := range featureVector.Values { + if featureVector.Statuses[idx] != serving.FieldStatus_PRESENT { + continue + } + featureValues[idx] = &prototypes.Value{Val: featureValue.Val} + } + for idx, ts := range featureVector.EventTimestamps { + if featureVector.Statuses[idx] != serving.FieldStatus_PRESENT { + continue + } + eventTimestamps[idx] = ×tamp.Timestamp{Seconds: ts.Seconds, Nanos: ts.Nanos} + } + // log.Println(request.Entities) + // for idx, featureName := range onlineResponse.Metadata.FeatureNames.Val { + // if _, ok := request.Entities[featureName]; ok { + // entityNames = append(entityNames, featureName) + // entityValues = append(entityValues, featureVector.Values[idx]) + // } else { + // featureNames = append(featureNames, onlineResponse.Metadata.FeatureNames.Val[idx:]...) + // featureValues = append(featureValues, featureVector.Values[idx:]...) + // featureStatuses = append(featureStatuses, featureVector.Statuses[idx:]...) + // eventTimestamps = append(eventTimestamps, featureVector.EventTimestamps[idx:]...) + // break + // } + // } + // featureNames = append(featureNames, onlineResponse.Metadata.FeatureNames.Val[:]...) + // featureValues = append(featureValues, featureVector.Values[:]...) + // featureStatuses = append(featureStatuses, featureVector.Statuses[:]...) + // eventTimestamps = append(eventTimestamps, featureVector.EventTimestamps[:]...) + + newLog := Log{ + featureNames: onlineResponse.Metadata.FeatureNames.Val, + featureValues: featureValues, + featureStatuses: featureVector.Statuses, + eventTimestamps: eventTimestamps, + // TODO(kevjumba): figure out if this is required + RequestContext: request.RequestContext, + } + s.logChannel <- newLog + } +} From c6656d451b4181f782a5bd42e8bb203bcea43bfb Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Thu, 17 Mar 2022 13:24:58 -0700 Subject: [PATCH 04/97] refactor Signed-off-by: Kevin Zhang --- go/cmd/server/logging.go | 91 ++++++++++++++++++++++++++++++++++++++++ go/cmd/server/main.go | 36 +++------------- go/cmd/server/server.go | 61 --------------------------- 3 files changed, 97 insertions(+), 91 deletions(-) create mode 100644 go/cmd/server/logging.go diff --git a/go/cmd/server/logging.go b/go/cmd/server/logging.go new file mode 100644 index 00000000000..dea5b4dbd77 --- /dev/null +++ b/go/cmd/server/logging.go @@ -0,0 +1,91 @@ +package main + +import ( + "log" + "time" + + "github.com/feast-dev/feast/go/protos/feast/serving" + "github.com/feast-dev/feast/go/protos/feast/types" + "google.golang.org/protobuf/types/known/timestamppb" +) + +type Log struct { + // Example: driver_id, customer_id + entityNames []string + // Example: val{int64_val: 5017}, val{int64_val: 1003} + entityValues []*types.Value + + // Feature names is 1:1 correspondence with featureValue, featureStatus, and timestamp + featureNames []string + + featureValues []*types.Value + featureStatuses []serving.FieldStatus + eventTimestamps []*timestamppb.Timestamp + RequestContext map[string]*types.RepeatedValue +} + +type MemoryBuffer struct { + logs []Log +} + +func flushToChannel(s *servingServiceServer, onlineResponse *serving.GetOnlineFeaturesResponse, request *serving.GetOnlineFeaturesRequest) { + for _, featureVector := range onlineResponse.Results { + featureValues := make([]*types.Value, len(featureVector.Values)) + eventTimestamps := make([]*timestamppb.Timestamp, len(featureVector.EventTimestamps)) + for idx, featureValue := range featureVector.Values { + if featureVector.Statuses[idx] != serving.FieldStatus_PRESENT { + continue + } + featureValues[idx] = &types.Value{Val: featureValue.Val} + } + for idx, ts := range featureVector.EventTimestamps { + if featureVector.Statuses[idx] != serving.FieldStatus_PRESENT { + continue + } + eventTimestamps[idx] = ×tamppb.Timestamp{Seconds: ts.Seconds, Nanos: ts.Nanos} + } + // log.Println(request.Entities) + // for idx, featureName := range onlineResponse.Metadata.FeatureNames.Val { + // if _, ok := request.Entities[featureName]; ok { + // entityNames = append(entityNames, featureName) + // entityValues = append(entityValues, featureVector.Values[idx]) + // } else { + // featureNames = append(featureNames, onlineResponse.Metadata.FeatureNames.Val[idx:]...) + // featureValues = append(featureValues, featureVector.Values[idx:]...) + // featureStatuses = append(featureStatuses, featureVector.Statuses[idx:]...) + // eventTimestamps = append(eventTimestamps, featureVector.EventTimestamps[idx:]...) + // break + // } + // } + // featureNames = append(featureNames, onlineResponse.Metadata.FeatureNames.Val[:]...) + // featureValues = append(featureValues, featureVector.Values[:]...) + // featureStatuses = append(featureStatuses, featureVector.Statuses[:]...) + // eventTimestamps = append(eventTimestamps, featureVector.EventTimestamps[:]...) + + newLog := Log{ + featureNames: onlineResponse.Metadata.FeatureNames.Val, + featureValues: featureValues, + featureStatuses: featureVector.Statuses, + eventTimestamps: eventTimestamps, + // TODO(kevjumba): figure out if this is required + RequestContext: request.RequestContext, + } + s.logChannel <- newLog + } +} + +func processLogs(log_channel chan Log, logBuffer *MemoryBuffer) { + // start a periodic flush + ticker := time.NewTicker(100 * time.Millisecond) + defer ticker.Stop() + for { + select { + case t := <-ticker.C: + log.Printf("time is %d", t) + log.Printf("Flushing buffer to offline storage with channel length: %d\n", len(logBuffer.logs)) + case new_log := <-log_channel: + log.Printf("Pushing %s to memory.\n", new_log.featureValues) + logBuffer.logs = append(logBuffer.logs, new_log) + } + } +} diff --git a/go/cmd/server/main.go b/go/cmd/server/main.go index 0b5aae456ae..22e125ca170 100644 --- a/go/cmd/server/main.go +++ b/go/cmd/server/main.go @@ -4,7 +4,7 @@ import ( "fmt" "log" "net" - "time" + "os" "github.com/feast-dev/feast/go/internal/feast" "github.com/feast-dev/feast/go/internal/feast/registry" @@ -13,7 +13,6 @@ import ( "github.com/feast-dev/feast/go/internal/feast" "github.com/feast-dev/feast/go/protos/feast/serving" - "github.com/kelseyhightower/envconfig" "google.golang.org/grpc" ) @@ -30,10 +29,6 @@ type FeastEnvConfig struct { SockFile string `envconfig:"FEAST_GRPC_SOCK_FILE"` } -type MemoryBuffer struct { - logs []Log -} - // TODO: Add a proper logging library such as https://github.com/Sirupsen/logrus func main() { // TODO(kevjumba) Figure out how large this log channel should be. @@ -43,13 +38,10 @@ func main() { logs: make([]Log, 0), } - var feastEnvConfig FeastEnvConfig - var err error - err = envconfig.Process("feast", &feastEnvConfig) - if err != nil { - log.Fatal(err) - } - if feastEnvConfig.RepoPath == "" && feastEnvConfig.RepoConfig == "" { + repoPath := os.Getenv(flagFeastRepoPath) + repoConfigJSON := os.Getenv(flagFeastRepoConfig) + sockFile := os.Getenv(flagFeastSockFile) + if repoPath == "" && repoConfigJSON == "" { log.Fatalln(fmt.Sprintf("One of %s of %s environment variables must be set", flagFeastRepoPath, flagFeastRepoConfig)) } @@ -73,7 +65,7 @@ func main() { log.Fatalln(err) } defer fs.DestructOnlineStore() - startGrpcServer(fs, logChannel, &logBuffer, feastEnvConfig.SockFile) + startGrpcServer(fs, logChannel, &logBuffer, sockFile) } func startGrpcServer(fs *feast.FeatureStore, logChannel chan Log, logBuffer *MemoryBuffer, sockFile string) { @@ -92,19 +84,3 @@ func startGrpcServer(fs *feast.FeatureStore, logChannel chan Log, logBuffer *Mem log.Fatalln(err) } } - -func processLogs(log_channel chan Log, logBuffer *MemoryBuffer) { - // start a periodic flush - ticker := time.NewTicker(100 * time.Millisecond) - defer ticker.Stop() - for { - select { - case t := <-ticker.C: - log.Printf("time is %d", t) - log.Printf("Flushing buffer to offline storage with channel length: %d\n", len(logBuffer.logs)) - case new_log := <-log_channel: - log.Printf("Pushing %s to memory.\n", new_log.featureValues) - logBuffer.logs = append(logBuffer.logs, new_log) - } - } -} diff --git a/go/cmd/server/server.go b/go/cmd/server/server.go index 520e8d72be2..2f22ebf6343 100644 --- a/go/cmd/server/server.go +++ b/go/cmd/server/server.go @@ -14,21 +14,6 @@ type servingServiceServer struct { serving.UnimplementedServingServiceServer } -type Log struct { - // Example: driver_id, customer_id - entityNames []string - // Example: val{int64_val: 5017}, val{int64_val: 1003} - entityValues []*prototypes.Value - - // Feature names is 1:1 correspondence with featureValue, featureStatus, and timestamp - featureNames []string - - featureValues []*prototypes.Value - featureStatuses []serving.FieldStatus - eventTimestamps []*timestamp.Timestamp - RequestContext map[string]*prototypes.RepeatedValue -} - func newServingServiceServer(fs *feast.FeatureStore, logChannel chan Log) *servingServiceServer { return &servingServiceServer{fs: fs, logChannel: logChannel} } @@ -83,49 +68,3 @@ func (s *servingServiceServer) GetOnlineFeatures(ctx context.Context, request *s return resp, nil } - -func flushToChannel(s *servingServiceServer, onlineResponse *serving.GetOnlineFeaturesResponse, request *serving.GetOnlineFeaturesRequest) { - for _, featureVector := range onlineResponse.Results { - featureValues := make([]*prototypes.Value, len(featureVector.Values)) - eventTimestamps := make([]*timestamp.Timestamp, len(featureVector.EventTimestamps)) - for idx, featureValue := range featureVector.Values { - if featureVector.Statuses[idx] != serving.FieldStatus_PRESENT { - continue - } - featureValues[idx] = &prototypes.Value{Val: featureValue.Val} - } - for idx, ts := range featureVector.EventTimestamps { - if featureVector.Statuses[idx] != serving.FieldStatus_PRESENT { - continue - } - eventTimestamps[idx] = ×tamp.Timestamp{Seconds: ts.Seconds, Nanos: ts.Nanos} - } - // log.Println(request.Entities) - // for idx, featureName := range onlineResponse.Metadata.FeatureNames.Val { - // if _, ok := request.Entities[featureName]; ok { - // entityNames = append(entityNames, featureName) - // entityValues = append(entityValues, featureVector.Values[idx]) - // } else { - // featureNames = append(featureNames, onlineResponse.Metadata.FeatureNames.Val[idx:]...) - // featureValues = append(featureValues, featureVector.Values[idx:]...) - // featureStatuses = append(featureStatuses, featureVector.Statuses[idx:]...) - // eventTimestamps = append(eventTimestamps, featureVector.EventTimestamps[idx:]...) - // break - // } - // } - // featureNames = append(featureNames, onlineResponse.Metadata.FeatureNames.Val[:]...) - // featureValues = append(featureValues, featureVector.Values[:]...) - // featureStatuses = append(featureStatuses, featureVector.Statuses[:]...) - // eventTimestamps = append(eventTimestamps, featureVector.EventTimestamps[:]...) - - newLog := Log{ - featureNames: onlineResponse.Metadata.FeatureNames.Val, - featureValues: featureValues, - featureStatuses: featureVector.Statuses, - eventTimestamps: eventTimestamps, - // TODO(kevjumba): figure out if this is required - RequestContext: request.RequestContext, - } - s.logChannel <- newLog - } -} From 17c81683aa8aac58c056073610014adb9560110f Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Thu, 17 Mar 2022 13:28:43 -0700 Subject: [PATCH 05/97] Add time Signed-off-by: Kevin Zhang --- go/cmd/server/logging.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/go/cmd/server/logging.go b/go/cmd/server/logging.go index dea5b4dbd77..08ac529b0f1 100644 --- a/go/cmd/server/logging.go +++ b/go/cmd/server/logging.go @@ -81,8 +81,7 @@ func processLogs(log_channel chan Log, logBuffer *MemoryBuffer) { for { select { case t := <-ticker.C: - log.Printf("time is %d", t) - log.Printf("Flushing buffer to offline storage with channel length: %d\n", len(logBuffer.logs)) + log.Printf("Flushing buffer to offline storage with channel length: %d\n at time %t", len(logBuffer.logs), t) case new_log := <-log_channel: log.Printf("Pushing %s to memory.\n", new_log.featureValues) logBuffer.logs = append(logBuffer.logs, new_log) From a4efbcea97e4b5e00e733e54c5d646afa9ff7e0b Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Thu, 17 Mar 2022 13:31:23 -0700 Subject: [PATCH 06/97] Add time Signed-off-by: Kevin Zhang --- go/cmd/server/logging.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/cmd/server/logging.go b/go/cmd/server/logging.go index 08ac529b0f1..05d59f4409b 100644 --- a/go/cmd/server/logging.go +++ b/go/cmd/server/logging.go @@ -81,7 +81,7 @@ func processLogs(log_channel chan Log, logBuffer *MemoryBuffer) { for { select { case t := <-ticker.C: - log.Printf("Flushing buffer to offline storage with channel length: %d\n at time %t", len(logBuffer.logs), t) + log.Printf("Flushing buffer to offline storage with channel length: %d\n at time: "+t.String(), len(logBuffer.logs)) case new_log := <-log_channel: log.Printf("Pushing %s to memory.\n", new_log.featureValues) logBuffer.logs = append(logBuffer.logs, new_log) From 79d8ba7388d05f3e7e27a2508a73142d1a173210 Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Thu, 17 Mar 2022 15:35:57 -0700 Subject: [PATCH 07/97] clean up Signed-off-by: Kevin Zhang --- go/cmd/server/main.go | 4 ---- go/internal/feast/featurestore_test.go | 8 ++++++++ 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/go/cmd/server/main.go b/go/cmd/server/main.go index 22e125ca170..612ae6e0431 100644 --- a/go/cmd/server/main.go +++ b/go/cmd/server/main.go @@ -10,10 +10,6 @@ import ( "github.com/feast-dev/feast/go/internal/feast/registry" "github.com/feast-dev/feast/go/protos/feast/serving" "google.golang.org/grpc" - - "github.com/feast-dev/feast/go/internal/feast" - "github.com/feast-dev/feast/go/protos/feast/serving" - "google.golang.org/grpc" ) const ( diff --git a/go/internal/feast/featurestore_test.go b/go/internal/feast/featurestore_test.go index 80b223f65e0..b67f9a4983b 100644 --- a/go/internal/feast/featurestore_test.go +++ b/go/internal/feast/featurestore_test.go @@ -49,11 +49,15 @@ func TestNewFeatureStore(t *testing.T) { } func TestGetOnlineFeaturesRedis(t *testing.T) { +<<<<<<< HEAD <<<<<<< HEAD t.Skip("@todo(achals): feature_repo isn't checked in yet") config := registry.RepoConfig{ ======= //t.Skip("@todo(achals): feature_repo isn't checked in yet") +======= + t.Skip("@todo(achals): feature_repo isn't checked in yet") +>>>>>>> df2b4a9b (clean up) config := RepoConfig{ >>>>>>> f7354334 (Make a proof of concept) Project: "feature_repo", @@ -74,7 +78,11 @@ func TestGetOnlineFeaturesRedis(t *testing.T) { {Val: &types.Value_Int64Val{Int64Val: 1003}}}}, } +<<<<<<< HEAD fs, err := NewFeatureStore(&config, nil) +======= + fs, err := NewFeatureStore(&config) +>>>>>>> df2b4a9b (clean up) assert.Nil(t, err) ctx := context.Background() response, err := fs.GetOnlineFeatures( From cd652bce0a8db846a88223336a8e2853b1e7ec37 Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Thu, 17 Mar 2022 15:38:31 -0700 Subject: [PATCH 08/97] Add comment Signed-off-by: Kevin Zhang --- go/cmd/server/logging.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/cmd/server/logging.go b/go/cmd/server/logging.go index 05d59f4409b..7c599b16f30 100644 --- a/go/cmd/server/logging.go +++ b/go/cmd/server/logging.go @@ -44,7 +44,7 @@ func flushToChannel(s *servingServiceServer, onlineResponse *serving.GetOnlineFe } eventTimestamps[idx] = ×tamppb.Timestamp{Seconds: ts.Seconds, Nanos: ts.Nanos} } - // log.Println(request.Entities) + // TODO(kevjumba): For filtering out and extracting entity names when the bug with join keys is fixed. // for idx, featureName := range onlineResponse.Metadata.FeatureNames.Val { // if _, ok := request.Entities[featureName]; ok { // entityNames = append(entityNames, featureName) From 4a13e8e1983d365c3140517758d3069087aa29f7 Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Thu, 17 Mar 2022 15:56:13 -0700 Subject: [PATCH 09/97] Add pseudocode Signed-off-by: Kevin Zhang --- go/cmd/server/logging.go | 15 ++++++++++++++- go/cmd/server/main.go | 2 +- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/go/cmd/server/logging.go b/go/cmd/server/logging.go index 7c599b16f30..a97bae4402c 100644 --- a/go/cmd/server/logging.go +++ b/go/cmd/server/logging.go @@ -4,6 +4,7 @@ import ( "log" "time" + "github.com/feast-dev/feast/go/internal/feast" "github.com/feast-dev/feast/go/protos/feast/serving" "github.com/feast-dev/feast/go/protos/feast/types" "google.golang.org/protobuf/types/known/timestamppb" @@ -74,7 +75,7 @@ func flushToChannel(s *servingServiceServer, onlineResponse *serving.GetOnlineFe } } -func processLogs(log_channel chan Log, logBuffer *MemoryBuffer) { +func processLogs(fs *feast.FeatureStore, log_channel chan Log, logBuffer *MemoryBuffer) { // start a periodic flush ticker := time.NewTicker(100 * time.Millisecond) defer ticker.Stop() @@ -82,9 +83,21 @@ func processLogs(log_channel chan Log, logBuffer *MemoryBuffer) { select { case t := <-ticker.C: log.Printf("Flushing buffer to offline storage with channel length: %d\n at time: "+t.String(), len(logBuffer.logs)) + flushLogsToOfflineStorage(fs, logBuffer) case new_log := <-log_channel: log.Printf("Pushing %s to memory.\n", new_log.featureValues) logBuffer.logs = append(logBuffer.logs, new_log) } } } + +func flushLogsToOfflineStorage(fs *feast.FeatureStore, logBuffer *MemoryBuffer) { + //offlineStore := fs.config.OfflineStore["type"] + // switch offlineStore{ + // case "file": + // // call python?? + // case "snowflake": + // + // } + //Do different row level manipulations and add to offline store +} diff --git a/go/cmd/server/main.go b/go/cmd/server/main.go index 612ae6e0431..9ebdf9d3ac5 100644 --- a/go/cmd/server/main.go +++ b/go/cmd/server/main.go @@ -65,7 +65,7 @@ func main() { } func startGrpcServer(fs *feast.FeatureStore, logChannel chan Log, logBuffer *MemoryBuffer, sockFile string) { - go processLogs(logChannel, logBuffer) + go processLogs(fs, logChannel, logBuffer) server := newServingServiceServer(fs, logChannel) log.Printf("Starting a gRPC server listening on %s\n", sockFile) lis, err := net.Listen("unix", sockFile) From a3f838432aeabe451612fd67a275108a91a9ffa5 Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Sun, 20 Mar 2022 14:03:07 -0700 Subject: [PATCH 10/97] Refactor logging functionality to hide internals Signed-off-by: Kevin Zhang --- go/cmd/server/logging.go | 36 ++++++++++++++----- go/cmd/server/main.go | 11 +++--- go/cmd/server/server.go | 9 +++-- .../online_store/test_universal_online.py | 2 ++ 4 files changed, 38 insertions(+), 20 deletions(-) diff --git a/go/cmd/server/logging.go b/go/cmd/server/logging.go index a97bae4402c..32c64c05f7d 100644 --- a/go/cmd/server/logging.go +++ b/go/cmd/server/logging.go @@ -26,7 +26,26 @@ type Log struct { } type MemoryBuffer struct { - logs []Log + logs []*Log +} + +type LoggingService struct { + memoryBuffer *MemoryBuffer + logChannel chan Log + fs *feast.FeatureStore +} + +func NewLoggingService(fs *feast.FeatureStore) *LoggingService { + // start handler processes? + loggingService := &LoggingService{ + logChannel: make(chan Log, 1000), + memoryBuffer: &MemoryBuffer{ + logs: make([]*Log, 0), + }, + fs: fs, + } + go loggingService.processLogs() + return loggingService } func flushToChannel(s *servingServiceServer, onlineResponse *serving.GetOnlineFeaturesResponse, request *serving.GetOnlineFeaturesRequest) { @@ -71,27 +90,27 @@ func flushToChannel(s *servingServiceServer, onlineResponse *serving.GetOnlineFe // TODO(kevjumba): figure out if this is required RequestContext: request.RequestContext, } - s.logChannel <- newLog + s.loggingService.logChannel <- newLog } } -func processLogs(fs *feast.FeatureStore, log_channel chan Log, logBuffer *MemoryBuffer) { +func (s *LoggingService) processLogs() { // start a periodic flush ticker := time.NewTicker(100 * time.Millisecond) defer ticker.Stop() for { select { case t := <-ticker.C: - log.Printf("Flushing buffer to offline storage with channel length: %d\n at time: "+t.String(), len(logBuffer.logs)) - flushLogsToOfflineStorage(fs, logBuffer) - case new_log := <-log_channel: + s.flushLogsToOfflineStorage(t) + case new_log := <-s.logChannel: log.Printf("Pushing %s to memory.\n", new_log.featureValues) - logBuffer.logs = append(logBuffer.logs, new_log) + s.memoryBuffer.logs = append(s.memoryBuffer.logs, &new_log) + time.Sleep(110 * time.Millisecond) } } } -func flushLogsToOfflineStorage(fs *feast.FeatureStore, logBuffer *MemoryBuffer) { +func (s *LoggingService) flushLogsToOfflineStorage(t time.Time) { //offlineStore := fs.config.OfflineStore["type"] // switch offlineStore{ // case "file": @@ -100,4 +119,5 @@ func flushLogsToOfflineStorage(fs *feast.FeatureStore, logBuffer *MemoryBuffer) // // } //Do different row level manipulations and add to offline store + log.Printf("Flushing buffer to offline storage with channel length: %d\n at time: "+t.String(), len(s.memoryBuffer.logs)) } diff --git a/go/cmd/server/main.go b/go/cmd/server/main.go index 9ebdf9d3ac5..252f288b893 100644 --- a/go/cmd/server/main.go +++ b/go/cmd/server/main.go @@ -30,9 +30,6 @@ func main() { // TODO(kevjumba) Figure out how large this log channel should be. logChannel := make(chan Log, 1000) defer close(logChannel) - var logBuffer = MemoryBuffer{ - logs: make([]Log, 0), - } repoPath := os.Getenv(flagFeastRepoPath) repoConfigJSON := os.Getenv(flagFeastRepoConfig) @@ -57,16 +54,16 @@ func main() { log.Println("Initializing feature store...") fs, err := feast.NewFeatureStore(repoConfig, nil) + loggingService := NewLoggingService(fs) if err != nil { log.Fatalln(err) } defer fs.DestructOnlineStore() - startGrpcServer(fs, logChannel, &logBuffer, sockFile) + startGrpcServer(fs, loggingService, sockFile) } -func startGrpcServer(fs *feast.FeatureStore, logChannel chan Log, logBuffer *MemoryBuffer, sockFile string) { - go processLogs(fs, logChannel, logBuffer) - server := newServingServiceServer(fs, logChannel) +func startGrpcServer(fs *feast.FeatureStore, loggingService *LoggingService, sockFile string) { + server := newServingServiceServer(fs, loggingService) log.Printf("Starting a gRPC server listening on %s\n", sockFile) lis, err := net.Listen("unix", sockFile) if err != nil { diff --git a/go/cmd/server/server.go b/go/cmd/server/server.go index 2f22ebf6343..c95d44f1355 100644 --- a/go/cmd/server/server.go +++ b/go/cmd/server/server.go @@ -9,13 +9,13 @@ import ( ) type servingServiceServer struct { - fs *feast.FeatureStore - logChannel chan Log + fs *feast.FeatureStore + loggingService *LoggingService serving.UnimplementedServingServiceServer } -func newServingServiceServer(fs *feast.FeatureStore, logChannel chan Log) *servingServiceServer { - return &servingServiceServer{fs: fs, logChannel: logChannel} +func newServingServiceServer(fs *feast.FeatureStore, loggingService *LoggingService) *servingServiceServer { + return &servingServiceServer{fs: fs, loggingService: loggingService} } func (s *servingServiceServer) GetFeastServingInfo(ctx context.Context, request *serving.GetFeastServingInfoRequest) (*serving.GetFeastServingInfoResponse, error) { @@ -32,7 +32,6 @@ func (s *servingServiceServer) GetOnlineFeatures(ctx context.Context, request *s if err != nil { return nil, err } - featureVectors, err := s.fs.GetOnlineFeatures( ctx, featuresOrService.FeaturesRefs, diff --git a/sdk/python/tests/integration/online_store/test_universal_online.py b/sdk/python/tests/integration/online_store/test_universal_online.py index 6762f01a1f3..ab5722b9dae 100644 --- a/sdk/python/tests/integration/online_store/test_universal_online.py +++ b/sdk/python/tests/integration/online_store/test_universal_online.py @@ -862,6 +862,8 @@ def test_online_retrieval_with_go_server( origins_df, destinations_df, ) + time.sleep(5) + assert(False) def setup_feature_store(environment, go_data_sources): From 706ee806ac26d76ab047f2a25aa1b46ed3aa916b Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Sun, 20 Mar 2022 15:03:23 -0700 Subject: [PATCH 11/97] Refactor Signed-off-by: Kevin Zhang --- go/cmd/server/logging.go | 56 +++---------------- go/cmd/server/server.go | 48 ++++++++++++++++ .../online_store/test_universal_online.py | 2 - 3 files changed, 55 insertions(+), 51 deletions(-) diff --git a/go/cmd/server/logging.go b/go/cmd/server/logging.go index 32c64c05f7d..60643ca01e2 100644 --- a/go/cmd/server/logging.go +++ b/go/cmd/server/logging.go @@ -31,14 +31,14 @@ type MemoryBuffer struct { type LoggingService struct { memoryBuffer *MemoryBuffer - logChannel chan Log + logChannel chan *Log fs *feast.FeatureStore } func NewLoggingService(fs *feast.FeatureStore) *LoggingService { // start handler processes? loggingService := &LoggingService{ - logChannel: make(chan Log, 1000), + logChannel: make(chan *Log, 1000), memoryBuffer: &MemoryBuffer{ logs: make([]*Log, 0), }, @@ -48,50 +48,9 @@ func NewLoggingService(fs *feast.FeatureStore) *LoggingService { return loggingService } -func flushToChannel(s *servingServiceServer, onlineResponse *serving.GetOnlineFeaturesResponse, request *serving.GetOnlineFeaturesRequest) { - for _, featureVector := range onlineResponse.Results { - featureValues := make([]*types.Value, len(featureVector.Values)) - eventTimestamps := make([]*timestamppb.Timestamp, len(featureVector.EventTimestamps)) - for idx, featureValue := range featureVector.Values { - if featureVector.Statuses[idx] != serving.FieldStatus_PRESENT { - continue - } - featureValues[idx] = &types.Value{Val: featureValue.Val} - } - for idx, ts := range featureVector.EventTimestamps { - if featureVector.Statuses[idx] != serving.FieldStatus_PRESENT { - continue - } - eventTimestamps[idx] = ×tamppb.Timestamp{Seconds: ts.Seconds, Nanos: ts.Nanos} - } - // TODO(kevjumba): For filtering out and extracting entity names when the bug with join keys is fixed. - // for idx, featureName := range onlineResponse.Metadata.FeatureNames.Val { - // if _, ok := request.Entities[featureName]; ok { - // entityNames = append(entityNames, featureName) - // entityValues = append(entityValues, featureVector.Values[idx]) - // } else { - // featureNames = append(featureNames, onlineResponse.Metadata.FeatureNames.Val[idx:]...) - // featureValues = append(featureValues, featureVector.Values[idx:]...) - // featureStatuses = append(featureStatuses, featureVector.Statuses[idx:]...) - // eventTimestamps = append(eventTimestamps, featureVector.EventTimestamps[idx:]...) - // break - // } - // } - // featureNames = append(featureNames, onlineResponse.Metadata.FeatureNames.Val[:]...) - // featureValues = append(featureValues, featureVector.Values[:]...) - // featureStatuses = append(featureStatuses, featureVector.Statuses[:]...) - // eventTimestamps = append(eventTimestamps, featureVector.EventTimestamps[:]...) - - newLog := Log{ - featureNames: onlineResponse.Metadata.FeatureNames.Val, - featureValues: featureValues, - featureStatuses: featureVector.Statuses, - eventTimestamps: eventTimestamps, - // TODO(kevjumba): figure out if this is required - RequestContext: request.RequestContext, - } - s.loggingService.logChannel <- newLog - } +func (s *LoggingService) emitLog(log *Log) error { + s.logChannel <- log + return nil } func (s *LoggingService) processLogs() { @@ -101,11 +60,10 @@ func (s *LoggingService) processLogs() { for { select { case t := <-ticker.C: - s.flushLogsToOfflineStorage(t) + go s.flushLogsToOfflineStorage(t) case new_log := <-s.logChannel: log.Printf("Pushing %s to memory.\n", new_log.featureValues) - s.memoryBuffer.logs = append(s.memoryBuffer.logs, &new_log) - time.Sleep(110 * time.Millisecond) + s.memoryBuffer.logs = append(s.memoryBuffer.logs, new_log) } } } diff --git a/go/cmd/server/server.go b/go/cmd/server/server.go index c95d44f1355..3d8b70079b2 100644 --- a/go/cmd/server/server.go +++ b/go/cmd/server/server.go @@ -5,7 +5,9 @@ import ( "github.com/feast-dev/feast/go/internal/feast" "github.com/feast-dev/feast/go/protos/feast/serving" + prototypes "github.com/feast-dev/feast/go/protos/feast/types" "github.com/feast-dev/feast/go/types" + "google.golang.org/protobuf/types/known/timestamppb" ) type servingServiceServer struct { @@ -67,3 +69,49 @@ func (s *servingServiceServer) GetOnlineFeatures(ctx context.Context, request *s return resp, nil } + +func generateLogs(s *servingServiceServer, onlineResponse *serving.GetOnlineFeaturesResponse, request *serving.GetOnlineFeaturesRequest) { + for _, featureVector := range onlineResponse.Results { + featureValues := make([]*prototypes.Value, len(featureVector.Values)) + eventTimestamps := make([]*timestamppb.Timestamp, len(featureVector.EventTimestamps)) + for idx, featureValue := range featureVector.Values { + if featureVector.Statuses[idx] != serving.FieldStatus_PRESENT { + continue + } + featureValues[idx] = &prototypes.Value{Val: featureValue.Val} + } + for idx, ts := range featureVector.EventTimestamps { + if featureVector.Statuses[idx] != serving.FieldStatus_PRESENT { + continue + } + eventTimestamps[idx] = ×tamppb.Timestamp{Seconds: ts.Seconds, Nanos: ts.Nanos} + } + // TODO(kevjumba): For filtering out and extracting entity names when the bug with join keys is fixed. + // for idx, featureName := range onlineResponse.Metadata.FeatureNames.Val { + // if _, ok := request.Entities[featureName]; ok { + // entityNames = append(entityNames, featureName) + // entityValues = append(entityValues, featureVector.Values[idx]) + // } else { + // featureNames = append(featureNames, onlineResponse.Metadata.FeatureNames.Val[idx:]...) + // featureValues = append(featureValues, featureVector.Values[idx:]...) + // featureStatuses = append(featureStatuses, featureVector.Statuses[idx:]...) + // eventTimestamps = append(eventTimestamps, featureVector.EventTimestamps[idx:]...) + // break + // } + // } + // featureNames = append(featureNames, onlineResponse.Metadata.FeatureNames.Val[:]...) + // featureValues = append(featureValues, featureVector.Values[:]...) + // featureStatuses = append(featureStatuses, featureVector.Statuses[:]...) + // eventTimestamps = append(eventTimestamps, featureVector.EventTimestamps[:]...) + + newLog := Log{ + featureNames: onlineResponse.Metadata.FeatureNames.Val, + featureValues: featureValues, + featureStatuses: featureVector.Statuses, + eventTimestamps: eventTimestamps, + // TODO(kevjumba): figure out if this is required + RequestContext: request.RequestContext, + } + s.loggingService.emitLog(&newLog) + } +} diff --git a/sdk/python/tests/integration/online_store/test_universal_online.py b/sdk/python/tests/integration/online_store/test_universal_online.py index ab5722b9dae..6762f01a1f3 100644 --- a/sdk/python/tests/integration/online_store/test_universal_online.py +++ b/sdk/python/tests/integration/online_store/test_universal_online.py @@ -862,8 +862,6 @@ def test_online_retrieval_with_go_server( origins_df, destinations_df, ) - time.sleep(5) - assert(False) def setup_feature_store(environment, go_data_sources): From 1831003b9b0316554b4b2acc64bcb3d3f4e102ca Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Sun, 20 Mar 2022 15:07:33 -0700 Subject: [PATCH 12/97] Revert changes Signed-off-by: Kevin Zhang --- go/cmd/server/main.go | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/go/cmd/server/main.go b/go/cmd/server/main.go index 252f288b893..a09897af63d 100644 --- a/go/cmd/server/main.go +++ b/go/cmd/server/main.go @@ -19,18 +19,8 @@ const ( feastServerVersion = "0.18.0" ) -type FeastEnvConfig struct { - RepoPath string `envconfig:"FEAST_REPO_PATH"` - RepoConfig string `envconfig:"FEAST_REPO_CONFIG"` - SockFile string `envconfig:"FEAST_GRPC_SOCK_FILE"` -} - // TODO: Add a proper logging library such as https://github.com/Sirupsen/logrus func main() { - // TODO(kevjumba) Figure out how large this log channel should be. - logChannel := make(chan Log, 1000) - defer close(logChannel) - repoPath := os.Getenv(flagFeastRepoPath) repoConfigJSON := os.Getenv(flagFeastRepoConfig) sockFile := os.Getenv(flagFeastSockFile) From 9e669c361d2e1d31a7294c0377bd292e56f28831 Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Sun, 20 Mar 2022 16:33:44 -0700 Subject: [PATCH 13/97] Add tests Signed-off-by: Kevin Zhang --- go/cmd/server/logging_test.go | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 go/cmd/server/logging_test.go diff --git a/go/cmd/server/logging_test.go b/go/cmd/server/logging_test.go new file mode 100644 index 00000000000..af1bd1310c4 --- /dev/null +++ b/go/cmd/server/logging_test.go @@ -0,0 +1,30 @@ +package main + +import ( + "reflect" + "testing" + "time" + + "github.com/feast-dev/feast/go/protos/feast/serving" + "github.com/stretchr/testify/assert" + "google.golang.org/protobuf/types/known/timestamppb" +) + +func TestLoggingChannelToMemoryBuffer(t *testing.T) { + //Feature store is still not checked in so we can't create one. + loggingService := NewLoggingService(nil) + assert.Empty(t, loggingService.memoryBuffer.logs) + ts := timestamppb.New(time.Now()) + newLog := Log{ + featureNames: []string{"feature1", "feature2"}, + featureStatuses: []serving.FieldStatus{serving.FieldStatus_PRESENT}, + eventTimestamps: []*timestamppb.Timestamp{ts}, + } + loggingService.emitLog(&newLog) + // Wait for memory buffer flush + time.Sleep(20 * time.Millisecond) + assert.Len(t, loggingService.memoryBuffer.logs, 1) + assert.Len(t, loggingService.logChannel, 0) + assert.True(t, reflect.DeepEqual(loggingService.memoryBuffer.logs[0].featureNames, []string{"feature1", "feature2"})) + assert.True(t, reflect.DeepEqual(loggingService.memoryBuffer.logs[0].featureStatuses, []serving.FieldStatus{serving.FieldStatus_PRESENT})) +} From b972634123d548d11ada8b1e5ded5b490f61e74c Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Sun, 20 Mar 2022 17:03:52 -0700 Subject: [PATCH 14/97] Add new timeout test Signed-off-by: Kevin Zhang --- go/cmd/server/logging.go | 18 +++++++++++++----- go/cmd/server/logging_test.go | 30 ++++++++++++++++++++++++++++-- go/cmd/server/main.go | 2 +- go/cmd/server/server.go | 8 ++++++-- 4 files changed, 48 insertions(+), 10 deletions(-) diff --git a/go/cmd/server/logging.go b/go/cmd/server/logging.go index 60643ca01e2..13562758b46 100644 --- a/go/cmd/server/logging.go +++ b/go/cmd/server/logging.go @@ -1,6 +1,7 @@ package main import ( + "fmt" "log" "time" @@ -35,22 +36,29 @@ type LoggingService struct { fs *feast.FeatureStore } -func NewLoggingService(fs *feast.FeatureStore) *LoggingService { +func NewLoggingService(fs *feast.FeatureStore, logChannelCapacity int, startLogProcessing bool) *LoggingService { // start handler processes? loggingService := &LoggingService{ - logChannel: make(chan *Log, 1000), + logChannel: make(chan *Log, logChannelCapacity), memoryBuffer: &MemoryBuffer{ logs: make([]*Log, 0), }, fs: fs, } - go loggingService.processLogs() + // For testing purposes, so we can test timeouts. + if startLogProcessing { + go loggingService.processLogs() + } return loggingService } func (s *LoggingService) emitLog(log *Log) error { - s.logChannel <- log - return nil + select { + case s.logChannel <- log: + return nil + case <-time.After(20 * time.Millisecond): + return fmt.Errorf("could not add to log channel with capacity %d. Current log channel length is %d", cap(s.logChannel), len(s.logChannel)) + } } func (s *LoggingService) processLogs() { diff --git a/go/cmd/server/logging_test.go b/go/cmd/server/logging_test.go index af1bd1310c4..31c79590530 100644 --- a/go/cmd/server/logging_test.go +++ b/go/cmd/server/logging_test.go @@ -12,7 +12,7 @@ import ( func TestLoggingChannelToMemoryBuffer(t *testing.T) { //Feature store is still not checked in so we can't create one. - loggingService := NewLoggingService(nil) + loggingService := NewLoggingService(nil, 10, true) assert.Empty(t, loggingService.memoryBuffer.logs) ts := timestamppb.New(time.Now()) newLog := Log{ @@ -20,11 +20,37 @@ func TestLoggingChannelToMemoryBuffer(t *testing.T) { featureStatuses: []serving.FieldStatus{serving.FieldStatus_PRESENT}, eventTimestamps: []*timestamppb.Timestamp{ts}, } - loggingService.emitLog(&newLog) + err := loggingService.emitLog(&newLog) // Wait for memory buffer flush time.Sleep(20 * time.Millisecond) assert.Len(t, loggingService.memoryBuffer.logs, 1) assert.Len(t, loggingService.logChannel, 0) assert.True(t, reflect.DeepEqual(loggingService.memoryBuffer.logs[0].featureNames, []string{"feature1", "feature2"})) assert.True(t, reflect.DeepEqual(loggingService.memoryBuffer.logs[0].featureStatuses, []serving.FieldStatus{serving.FieldStatus_PRESENT})) + assert.Nil(t, err) +} + +func TestLoggingChannelTiemout(t *testing.T) { + //Feature store is still not checked in so we can't create one. + loggingService := NewLoggingService(nil, 1, false) + assert.Empty(t, loggingService.memoryBuffer.logs) + ts := timestamppb.New(time.Now()) + newLog := Log{ + featureNames: []string{"feature1", "feature2"}, + featureStatuses: []serving.FieldStatus{serving.FieldStatus_PRESENT}, + eventTimestamps: []*timestamppb.Timestamp{ts, ts}, + } + loggingService.emitLog(&newLog) + // Wait for memory buffer flush + time.Sleep(20 * time.Millisecond) + newTs := timestamppb.New(time.Now()) + + newLog2 := Log{ + featureNames: []string{"feature4", "feature5"}, + featureStatuses: []serving.FieldStatus{serving.FieldStatus_PRESENT}, + eventTimestamps: []*timestamppb.Timestamp{newTs, newTs}, + } + err := loggingService.emitLog(&newLog2) + time.Sleep(20 * time.Millisecond) + assert.NotNil(t, err) } diff --git a/go/cmd/server/main.go b/go/cmd/server/main.go index a09897af63d..4b380b27099 100644 --- a/go/cmd/server/main.go +++ b/go/cmd/server/main.go @@ -44,7 +44,7 @@ func main() { log.Println("Initializing feature store...") fs, err := feast.NewFeatureStore(repoConfig, nil) - loggingService := NewLoggingService(fs) + loggingService := NewLoggingService(fs, 1000, true) if err != nil { log.Fatalln(err) } diff --git a/go/cmd/server/server.go b/go/cmd/server/server.go index 3d8b70079b2..0d40e41ba84 100644 --- a/go/cmd/server/server.go +++ b/go/cmd/server/server.go @@ -70,7 +70,7 @@ func (s *servingServiceServer) GetOnlineFeatures(ctx context.Context, request *s return resp, nil } -func generateLogs(s *servingServiceServer, onlineResponse *serving.GetOnlineFeaturesResponse, request *serving.GetOnlineFeaturesRequest) { +func generateLogs(s *servingServiceServer, onlineResponse *serving.GetOnlineFeaturesResponse, request *serving.GetOnlineFeaturesRequest) error { for _, featureVector := range onlineResponse.Results { featureValues := make([]*prototypes.Value, len(featureVector.Values)) eventTimestamps := make([]*timestamppb.Timestamp, len(featureVector.EventTimestamps)) @@ -112,6 +112,10 @@ func generateLogs(s *servingServiceServer, onlineResponse *serving.GetOnlineFeat // TODO(kevjumba): figure out if this is required RequestContext: request.RequestContext, } - s.loggingService.emitLog(&newLog) + err := s.loggingService.emitLog(&newLog) + if err != nil { + return err + } } + return nil } From 42bdc8b89a2ef443c9c88d0a8aa811cbd6deaf14 Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Fri, 25 Mar 2022 10:19:14 -0700 Subject: [PATCH 15/97] Fix python ci for m1 mac Signed-off-by: Kevin Zhang --- go.mod | 10 ++++++++++ go.sum | 2 ++ sdk/python/requirements/py3.9-ci-requirements.txt | 2 +- 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index a42179f7bc3..2c3459384ab 100644 --- a/go.mod +++ b/go.mod @@ -41,6 +41,16 @@ require ( golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3 // indirect golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd // indirect golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27 // indirect + github.com/google/flatbuffers v2.0.0+incompatible // indirect + github.com/google/go-cmp v0.5.7 // indirect + github.com/klauspost/compress v1.15.1 // indirect + github.com/pierrec/lz4/v4 v4.1.9 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + golang.org/x/exp v0.0.0-20211028214138-64b4c8e87d1a // indirect + golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3 // indirect + golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f // indirect + golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359 // indirect golang.org/x/text v0.3.7 // indirect golang.org/x/tools v0.1.10 // indirect golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect diff --git a/go.sum b/go.sum index 0ee1c0b9c5f..b17f01aa905 100644 --- a/go.sum +++ b/go.sum @@ -429,6 +429,8 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= golang.org/x/mod v0.6.0-dev.0.20211013180041-c96bc1413d57 h1:LQmS1nU0twXLA96Kt7U9qtHJEbBk3z6Q0V4UXjZkpr4= golang.org/x/mod v0.6.0-dev.0.20211013180041-c96bc1413d57/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= +golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.5.1-0.20210830214625-1b1db11ec8f4/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3 h1:kQgndtyPBW/JIYERgdxfwMYh3AVStj88WQTlNDi2a+o= golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= diff --git a/sdk/python/requirements/py3.9-ci-requirements.txt b/sdk/python/requirements/py3.9-ci-requirements.txt index cf228b9412b..03f64cb404a 100644 --- a/sdk/python/requirements/py3.9-ci-requirements.txt +++ b/sdk/python/requirements/py3.9-ci-requirements.txt @@ -118,7 +118,7 @@ colorama==0.4.4 # via feast (setup.py) coverage[toml]==6.3.2 # via pytest-cov -cryptography==3.3.2 +cryptography==3.4.8 # via # adal # azure-identity From ab8d1e017da0e83cf4c2c3a7c2a523d8da030522 Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Mon, 28 Mar 2022 18:16:21 -0700 Subject: [PATCH 16/97] Fix lint Signed-off-by: Kevin Zhang --- go/internal/feast/featurestore_test.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/go/internal/feast/featurestore_test.go b/go/internal/feast/featurestore_test.go index b67f9a4983b..6b8052c8034 100644 --- a/go/internal/feast/featurestore_test.go +++ b/go/internal/feast/featurestore_test.go @@ -15,9 +15,6 @@ import ( "github.com/feast-dev/feast/go/protos/feast/types" "github.com/stretchr/testify/assert" - - "github.com/feast-dev/feast/go/protos/feast/types" - "github.com/stretchr/testify/assert" ) // Return absolute path to the test_repo registry regardless of the working directory From f6c2c8752591658da5a792e07be109f6318ee167 Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Tue, 29 Mar 2022 18:18:30 -0700 Subject: [PATCH 17/97] Working state Signed-off-by: Kevin Zhang --- go.mod | 3 - go.sum | 5 - go/cmd/server/{ => logging}/logging.go | 25 ++-- go/cmd/server/{ => logging}/logging_test.go | 32 +++--- go/cmd/server/main.go | 8 +- go/cmd/server/server.go | 107 ++++++++++-------- go/cmd/server/server_test.go | 43 ++++++- go/internal/feast/featurestore.go | 10 ++ go/internal/feast/offline_log_store.go | 27 +++++ go/internal/test/go_integration_test_utils.go | 1 + 10 files changed, 179 insertions(+), 82 deletions(-) rename go/cmd/server/{ => logging}/logging.go (80%) rename go/cmd/server/{ => logging}/logging_test.go (60%) create mode 100644 go/internal/feast/offline_log_store.go diff --git a/go.mod b/go.mod index 2c3459384ab..ad8f81ceeee 100644 --- a/go.mod +++ b/go.mod @@ -45,14 +45,11 @@ require ( github.com/google/go-cmp v0.5.7 // indirect github.com/klauspost/compress v1.15.1 // indirect github.com/pierrec/lz4/v4 v4.1.9 // indirect - github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect golang.org/x/exp v0.0.0-20211028214138-64b4c8e87d1a // indirect - golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3 // indirect golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f // indirect golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359 // indirect golang.org/x/text v0.3.7 // indirect - golang.org/x/tools v0.1.10 // indirect golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect google.golang.org/genproto v0.0.0-20220126215142-9970aeb2e350 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect diff --git a/go.sum b/go.sum index b17f01aa905..a75bd9d96fd 100644 --- a/go.sum +++ b/go.sum @@ -145,9 +145,7 @@ github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiu github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/gonuts/commander v0.1.0 h1:EcDTiVw9oAVORFjQOEOuHQqcl6OXMyTgELocTq6zJ0I= github.com/gonuts/commander v0.1.0/go.mod h1:qkb5mSlcWodYgo7vs8ulLnXhfinhZsZcm6+H/z1JjgY= -github.com/gonuts/flag v0.1.0 h1:fqMv/MZ+oNGu0i9gp0/IQ/ZaPIDoAZBOBaJoV7viCWM= github.com/gonuts/flag v0.1.0/go.mod h1:ZTmTGtrSPejTo/SRNhCqwLTmiAgyBdCkLYhHrAoBdz4= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= @@ -298,7 +296,6 @@ github.com/pierrec/lz4/v4 v4.1.12 h1:44l88ehTZAUGW4VlO1QC4zkilL99M6Y9MXNwEs0uzP8 github.com/pierrec/lz4/v4 v4.1.12/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/profile v1.2.1/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -431,8 +428,6 @@ golang.org/x/mod v0.6.0-dev.0.20211013180041-c96bc1413d57 h1:LQmS1nU0twXLA96Kt7U golang.org/x/mod v0.6.0-dev.0.20211013180041-c96bc1413d57/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.5.1-0.20210830214625-1b1db11ec8f4/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= -golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3 h1:kQgndtyPBW/JIYERgdxfwMYh3AVStj88WQTlNDi2a+o= -golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= diff --git a/go/cmd/server/logging.go b/go/cmd/server/logging/logging.go similarity index 80% rename from go/cmd/server/logging.go rename to go/cmd/server/logging/logging.go index 13562758b46..8fc265d05b8 100644 --- a/go/cmd/server/logging.go +++ b/go/cmd/server/logging/logging.go @@ -1,4 +1,4 @@ -package main +package logging import ( "fmt" @@ -13,16 +13,16 @@ import ( type Log struct { // Example: driver_id, customer_id - entityNames []string + EntityName string // Example: val{int64_val: 5017}, val{int64_val: 1003} - entityValues []*types.Value + EntityValue *types.Value // Feature names is 1:1 correspondence with featureValue, featureStatus, and timestamp - featureNames []string + FeatureNames []string - featureValues []*types.Value - featureStatuses []serving.FieldStatus - eventTimestamps []*timestamppb.Timestamp + FeatureValues []*types.Value + FeatureStatuses []serving.FieldStatus + EventTimestamps []*timestamppb.Timestamp RequestContext map[string]*types.RepeatedValue } @@ -52,7 +52,7 @@ func NewLoggingService(fs *feast.FeatureStore, logChannelCapacity int, startLogP return loggingService } -func (s *LoggingService) emitLog(log *Log) error { +func (s *LoggingService) EmitLog(log *Log) error { select { case s.logChannel <- log: return nil @@ -70,7 +70,7 @@ func (s *LoggingService) processLogs() { case t := <-ticker.C: go s.flushLogsToOfflineStorage(t) case new_log := <-s.logChannel: - log.Printf("Pushing %s to memory.\n", new_log.featureValues) + log.Printf("Pushing %s to memory.\n", new_log.FeatureValues) s.memoryBuffer.logs = append(s.memoryBuffer.logs, new_log) } } @@ -86,4 +86,11 @@ func (s *LoggingService) flushLogsToOfflineStorage(t time.Time) { // } //Do different row level manipulations and add to offline store log.Printf("Flushing buffer to offline storage with channel length: %d\n at time: "+t.String(), len(s.memoryBuffer.logs)) + if s.fs.GetRepoConfig().OfflineStore["type"] == "file" { + //file_path := s.fs.GetRepoConfig().RepoPath + + } else { + // Currently don't support any other offline flushing. + return + } } diff --git a/go/cmd/server/logging_test.go b/go/cmd/server/logging/logging_test.go similarity index 60% rename from go/cmd/server/logging_test.go rename to go/cmd/server/logging/logging_test.go index 31c79590530..46d44a9c101 100644 --- a/go/cmd/server/logging_test.go +++ b/go/cmd/server/logging/logging_test.go @@ -1,4 +1,4 @@ -package main +package logging import ( "reflect" @@ -16,41 +16,41 @@ func TestLoggingChannelToMemoryBuffer(t *testing.T) { assert.Empty(t, loggingService.memoryBuffer.logs) ts := timestamppb.New(time.Now()) newLog := Log{ - featureNames: []string{"feature1", "feature2"}, - featureStatuses: []serving.FieldStatus{serving.FieldStatus_PRESENT}, - eventTimestamps: []*timestamppb.Timestamp{ts}, + FeatureNames: []string{"feature1", "feature2"}, + FeatureStatuses: []serving.FieldStatus{serving.FieldStatus_PRESENT}, + EventTimestamps: []*timestamppb.Timestamp{ts}, } - err := loggingService.emitLog(&newLog) + err := loggingService.EmitLog(&newLog) // Wait for memory buffer flush time.Sleep(20 * time.Millisecond) assert.Len(t, loggingService.memoryBuffer.logs, 1) assert.Len(t, loggingService.logChannel, 0) - assert.True(t, reflect.DeepEqual(loggingService.memoryBuffer.logs[0].featureNames, []string{"feature1", "feature2"})) - assert.True(t, reflect.DeepEqual(loggingService.memoryBuffer.logs[0].featureStatuses, []serving.FieldStatus{serving.FieldStatus_PRESENT})) + assert.True(t, reflect.DeepEqual(loggingService.memoryBuffer.logs[0].FeatureNames, []string{"feature1", "feature2"})) + assert.True(t, reflect.DeepEqual(loggingService.memoryBuffer.logs[0].FeatureStatuses, []serving.FieldStatus{serving.FieldStatus_PRESENT})) assert.Nil(t, err) } -func TestLoggingChannelTiemout(t *testing.T) { +func TestLoggingChannelTimeout(t *testing.T) { //Feature store is still not checked in so we can't create one. loggingService := NewLoggingService(nil, 1, false) assert.Empty(t, loggingService.memoryBuffer.logs) ts := timestamppb.New(time.Now()) newLog := Log{ - featureNames: []string{"feature1", "feature2"}, - featureStatuses: []serving.FieldStatus{serving.FieldStatus_PRESENT}, - eventTimestamps: []*timestamppb.Timestamp{ts, ts}, + FeatureNames: []string{"feature1", "feature2"}, + FeatureStatuses: []serving.FieldStatus{serving.FieldStatus_PRESENT}, + EventTimestamps: []*timestamppb.Timestamp{ts, ts}, } - loggingService.emitLog(&newLog) + loggingService.EmitLog(&newLog) // Wait for memory buffer flush time.Sleep(20 * time.Millisecond) newTs := timestamppb.New(time.Now()) newLog2 := Log{ - featureNames: []string{"feature4", "feature5"}, - featureStatuses: []serving.FieldStatus{serving.FieldStatus_PRESENT}, - eventTimestamps: []*timestamppb.Timestamp{newTs, newTs}, + FeatureNames: []string{"feature4", "feature5"}, + FeatureStatuses: []serving.FieldStatus{serving.FieldStatus_PRESENT}, + EventTimestamps: []*timestamppb.Timestamp{newTs, newTs}, } - err := loggingService.emitLog(&newLog2) + err := loggingService.EmitLog(&newLog2) time.Sleep(20 * time.Millisecond) assert.NotNil(t, err) } diff --git a/go/cmd/server/main.go b/go/cmd/server/main.go index 4b380b27099..9404c998412 100644 --- a/go/cmd/server/main.go +++ b/go/cmd/server/main.go @@ -6,6 +6,7 @@ import ( "net" "os" + "github.com/feast-dev/feast/go/cmd/server/logging" "github.com/feast-dev/feast/go/internal/feast" "github.com/feast-dev/feast/go/internal/feast/registry" "github.com/feast-dev/feast/go/protos/feast/serving" @@ -43,8 +44,13 @@ func main() { } log.Println("Initializing feature store...") +<<<<<<< HEAD fs, err := feast.NewFeatureStore(repoConfig, nil) loggingService := NewLoggingService(fs, 1000, true) +======= + fs, err := feast.NewFeatureStore(repoConfig) + loggingService := logging.NewLoggingService(fs, 1000, true) +>>>>>>> 118a377d (Working state) if err != nil { log.Fatalln(err) } @@ -52,7 +58,7 @@ func main() { startGrpcServer(fs, loggingService, sockFile) } -func startGrpcServer(fs *feast.FeatureStore, loggingService *LoggingService, sockFile string) { +func startGrpcServer(fs *feast.FeatureStore, loggingService *logging.LoggingService, sockFile string) { server := newServingServiceServer(fs, loggingService) log.Printf("Starting a gRPC server listening on %s\n", sockFile) lis, err := net.Listen("unix", sockFile) diff --git a/go/cmd/server/server.go b/go/cmd/server/server.go index 0d40e41ba84..88e36608ca6 100644 --- a/go/cmd/server/server.go +++ b/go/cmd/server/server.go @@ -3,6 +3,7 @@ package main import ( "context" + "github.com/feast-dev/feast/go/cmd/server/logging" "github.com/feast-dev/feast/go/internal/feast" "github.com/feast-dev/feast/go/protos/feast/serving" prototypes "github.com/feast-dev/feast/go/protos/feast/types" @@ -12,11 +13,11 @@ import ( type servingServiceServer struct { fs *feast.FeatureStore - loggingService *LoggingService + loggingService *logging.LoggingService serving.UnimplementedServingServiceServer } -func newServingServiceServer(fs *feast.FeatureStore, loggingService *LoggingService) *servingServiceServer { +func newServingServiceServer(fs *feast.FeatureStore, loggingService *logging.LoggingService) *servingServiceServer { return &servingServiceServer{fs: fs, loggingService: loggingService} } @@ -51,68 +52,84 @@ func (s *servingServiceServer) GetOnlineFeatures(ctx context.Context, request *s FeatureNames: &serving.FeatureList{Val: make([]string, 0)}, }, } +<<<<<<< HEAD for _, vector := range featureVectors { +======= + // Entities are currently part of the features as a value + entityNames := make([]string, 0) + entityValues := make([]*prototypes.Value, 0) + for name, values := range request.Entities { + resp.Metadata.FeatureNames.Val = append(resp.Metadata.FeatureNames.Val, name) + vec := &serving.GetOnlineFeaturesResponse_FeatureVector{ + Values: make([]*prototypes.Value, 0), + Statuses: make([]serving.FieldStatus, 0), + EventTimestamps: make([]*timestamp.Timestamp, 0), + } + resp.Results = append(resp.Results, vec) + for _, v := range values.Val { + entityNames = append(entityNames, name) + entityValues = append(entityValues, v) + vec.Values = append(vec.Values, v) + vec.Statuses = append(vec.Statuses, serving.FieldStatus_PRESENT) + vec.EventTimestamps = append(vec.EventTimestamps, ×tamp.Timestamp{}) + } + } + featureNames := make([]string, len(featureVectors)) + for idx, vector := range featureVectors { +>>>>>>> 118a377d (Working state) resp.Metadata.FeatureNames.Val = append(resp.Metadata.FeatureNames.Val, vector.Name) - + featureNames[idx] = vector.Name values, err := types.ArrowValuesToProtoValues(vector.Values) if err != nil { return nil, err } - resp.Results = append(resp.Results, &serving.GetOnlineFeaturesResponse_FeatureVector{ Values: values, Statuses: vector.Statuses, EventTimestamps: vector.Timestamps, }) } - + go generateLogs(s, entityNames, entityValues, featureNames, resp.Results, request) return resp, nil } -func generateLogs(s *servingServiceServer, onlineResponse *serving.GetOnlineFeaturesResponse, request *serving.GetOnlineFeaturesRequest) error { - for _, featureVector := range onlineResponse.Results { - featureValues := make([]*prototypes.Value, len(featureVector.Values)) - eventTimestamps := make([]*timestamppb.Timestamp, len(featureVector.EventTimestamps)) - for idx, featureValue := range featureVector.Values { - if featureVector.Statuses[idx] != serving.FieldStatus_PRESENT { - continue - } - featureValues[idx] = &prototypes.Value{Val: featureValue.Val} - } - for idx, ts := range featureVector.EventTimestamps { - if featureVector.Statuses[idx] != serving.FieldStatus_PRESENT { - continue - } - eventTimestamps[idx] = ×tamppb.Timestamp{Seconds: ts.Seconds, Nanos: ts.Nanos} +func generateLogs(s *servingServiceServer, entityNames []string, entityValues []*prototypes.Value, featureNames []string, results []*serving.GetOnlineFeaturesResponse_FeatureVector, request *serving.GetOnlineFeaturesRequest) error { + // Add a log with the request context + if request.RequestContext != nil && len(request.RequestContext) > 0 { + requestContextLog := logging.Log{ + RequestContext: request.RequestContext, } - // TODO(kevjumba): For filtering out and extracting entity names when the bug with join keys is fixed. - // for idx, featureName := range onlineResponse.Metadata.FeatureNames.Val { - // if _, ok := request.Entities[featureName]; ok { - // entityNames = append(entityNames, featureName) - // entityValues = append(entityValues, featureVector.Values[idx]) - // } else { - // featureNames = append(featureNames, onlineResponse.Metadata.FeatureNames.Val[idx:]...) - // featureValues = append(featureValues, featureVector.Values[idx:]...) - // featureStatuses = append(featureStatuses, featureVector.Statuses[idx:]...) - // eventTimestamps = append(eventTimestamps, featureVector.EventTimestamps[idx:]...) - // break - // } - // } - // featureNames = append(featureNames, onlineResponse.Metadata.FeatureNames.Val[:]...) - // featureValues = append(featureValues, featureVector.Values[:]...) - // featureStatuses = append(featureStatuses, featureVector.Statuses[:]...) - // eventTimestamps = append(eventTimestamps, featureVector.EventTimestamps[:]...) + s.loggingService.EmitLog(&requestContextLog) + } - newLog := Log{ - featureNames: onlineResponse.Metadata.FeatureNames.Val, - featureValues: featureValues, - featureStatuses: featureVector.Statuses, - eventTimestamps: eventTimestamps, - // TODO(kevjumba): figure out if this is required - RequestContext: request.RequestContext, + if len(results) <= 0 { + return nil + } + numFeatures := len(featureNames) + numRows := len(results[0].Values) + featureValueLogRows := make([][]*prototypes.Value, numRows) + featureStatusLogRows := make([][]serving.FieldStatus, numRows) + eventTimestampLogRows := make([][]*timestamppb.Timestamp, numRows) + + for row_idx := 0; row_idx < numRows; row_idx++ { + featureValueLogRows[row_idx] = make([]*prototypes.Value, numFeatures) + featureStatusLogRows[row_idx] = make([]serving.FieldStatus, numFeatures) + eventTimestampLogRows[row_idx] = make([]*timestamppb.Timestamp, numFeatures) + for idx := 1; idx < len(results); idx++ { + featureValueLogRows[row_idx][idx-1] = results[idx].Values[row_idx] + featureStatusLogRows[row_idx][idx-1] = results[idx].Statuses[row_idx] + eventTimestampLogRows[row_idx][idx-1] = results[idx].EventTimestamps[row_idx] + } + newLog := logging.Log{ + EntityName: entityNames[row_idx], + EntityValue: entityValues[row_idx], + FeatureNames: featureNames, + FeatureValues: featureValueLogRows[row_idx], + FeatureStatuses: featureStatusLogRows[row_idx], + EventTimestamps: eventTimestampLogRows[row_idx], } - err := s.loggingService.emitLog(&newLog) + err := s.loggingService.EmitLog(&newLog) if err != nil { return err } diff --git a/go/cmd/server/server_test.go b/go/cmd/server/server_test.go index 730558f1b6d..354f6a60262 100644 --- a/go/cmd/server/server_test.go +++ b/go/cmd/server/server_test.go @@ -8,7 +8,9 @@ import ( "reflect" "runtime" "testing" + "time" + "github.com/feast-dev/feast/go/cmd/server/logging" "github.com/feast-dev/feast/go/internal/feast" "github.com/feast-dev/feast/go/internal/test" "github.com/feast-dev/feast/go/protos/feast/serving" @@ -46,7 +48,10 @@ func getClient(ctx context.Context, basePath string) (serving.ServingServiceClie if err != nil { panic(err) } - serving.RegisterServingServiceServer(server, &servingServiceServer{fs: fs}) + loggingService := logging.NewLoggingService(fs, 1000, true) + servingServiceServer := newServingServiceServer(fs, loggingService) + + serving.RegisterServingServiceServer(server, servingServiceServer) go func() { if err := server.Serve(listener); err != nil { panic(err) @@ -107,14 +112,14 @@ func TestGetOnlineFeaturesSqlite(t *testing.T) { Entities: entities, } response, err := client.GetOnlineFeatures(ctx, request) + assert.Nil(t, err) + assert.NotNil(t, response) expectedEntityValuesResp := []*types.Value{ {Val: &types.Value_Int64Val{Int64Val: 1001}}, {Val: &types.Value_Int64Val{Int64Val: 1003}}, {Val: &types.Value_Int64Val{Int64Val: 1005}}, } expectedFeatureNamesResp := []string{"driver_id", "conv_rate", "acc_rate", "avg_daily_trips"} - assert.Nil(t, err) - assert.NotNil(t, response) rows, err := test.ReadParquet(filepath.Join(dir, "feature_repo", "data", "driver_stats.parquet")) assert.Nil(t, err) entityKeys := map[int64]bool{1001: true, 1003: true, 1005: true} @@ -137,4 +142,36 @@ func TestGetOnlineFeaturesSqlite(t *testing.T) { assert.True(t, reflect.DeepEqual(response.Results[3].Values, expectedAvgDailyTripsValues)) assert.True(t, reflect.DeepEqual(response.Metadata.FeatureNames.Val, expectedFeatureNamesResp)) + time.Sleep(1 * time.Second) +} + +func TestGetOnlineFeaturesSqliteWithLogging(t *testing.T) { + ctx := context.Background() + // Pregenerated using `feast init`. + dir := "." + err := test.SetupFeatureRepo(dir) + assert.Nil(t, err) + defer test.CleanUpRepo(dir) + client, closer := getClient(ctx, dir) + defer closer() + entities := make(map[string]*types.RepeatedValue) + entities["driver_id"] = &types.RepeatedValue{ + Val: []*types.Value{ + {Val: &types.Value_Int64Val{Int64Val: 1001}}, + {Val: &types.Value_Int64Val{Int64Val: 1003}}, + {Val: &types.Value_Int64Val{Int64Val: 1005}}, + }, + } + request := &serving.GetOnlineFeaturesRequest{ + Kind: &serving.GetOnlineFeaturesRequest_Features{ + Features: &serving.FeatureList{ + Val: []string{"driver_hourly_stats:conv_rate", "driver_hourly_stats:acc_rate", "driver_hourly_stats:avg_daily_trips"}, + }, + }, + Entities: entities, + } + response, err := client.GetOnlineFeatures(ctx, request) + assert.Nil(t, err) + assert.NotNil(t, response) + } diff --git a/go/internal/feast/featurestore.go b/go/internal/feast/featurestore.go index 3b866f143dc..4cefa9fe2fa 100644 --- a/go/internal/feast/featurestore.go +++ b/go/internal/feast/featurestore.go @@ -32,8 +32,14 @@ type Features struct { // NewFeatureStore constructs a feature store fat client using the // repo config (contents of feature_store.yaml converted to JSON map). +<<<<<<< HEAD func NewFeatureStore(config *registry.RepoConfig, callback transformation.TransformationCallback) (*FeatureStore, error) { onlineStore, err := onlinestore.NewOnlineStore(config) +======= +func NewFeatureStore(config *RepoConfig) (*FeatureStore, error) { + onlineStore, err := NewOnlineStore(config) + // offlineStore, err := NewOfflineStore(config) +>>>>>>> 118a377d (Working state) if err != nil { return nil, err } @@ -185,6 +191,10 @@ func (fs *FeatureStore) DestructOnlineStore() { fs.onlineStore.Destruct() } +func (fs *FeatureStore) GetRepoConfig() *RepoConfig { + return fs.config +} + // ParseFeatures parses the kind field of a GetOnlineFeaturesRequest protobuf message // and populates a Features struct with the result. func (fs *FeatureStore) ParseFeatures(kind interface{}) (*Features, error) { diff --git a/go/internal/feast/offline_log_store.go b/go/internal/feast/offline_log_store.go new file mode 100644 index 00000000000..9f2c45b440f --- /dev/null +++ b/go/internal/feast/offline_log_store.go @@ -0,0 +1,27 @@ +package feast + +// import "github.com/feast-dev/feast/go/cmd/server/logging" + +// type OfflineLogStorage interface { +// FlushToStorage(logging.MemoryBuffer) +// // Destruct must be call once user is done using OnlineStore +// // This is to comply with the Connector since we have to close the plugin +// } + +// func getOfflineStoreType(offlineStoreConfig map[string]interface{}) (string, bool) { +// if onlineStoreType, ok := offlineStoreConfig["type"]; !ok { +// // Assume file for case of no specified. +// return "file", true +// } else { +// result, ok := onlineStoreType.(string) +// return result, ok +// } +// } + +// func NewOfflineStore(config *RepoConfig) (OfflineLogStorage, error) { +// onlineStoreType, ok := getOfflineStoreType(config.OfflineStore) +// if !ok { +// onlineStore, err := NewSqliteOnlineStore(config.Project, config, config.OnlineStore) +// return onlineStore, err +// } +// } diff --git a/go/internal/test/go_integration_test_utils.go b/go/internal/test/go_integration_test_utils.go index dcaa1bc3d5d..154bf36ff86 100644 --- a/go/internal/test/go_integration_test_utils.go +++ b/go/internal/test/go_integration_test_utils.go @@ -124,6 +124,7 @@ func CleanUpRepo(basePath string) error { } err = os.RemoveAll(feature_repo_path) if err != nil { + log.Fatalf("Couldn't remove feature repo path. %s", err) return err } return nil From b6b412d19a3e6d2af876f3e5fffd14611ca773be Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Wed, 30 Mar 2022 11:17:58 -0700 Subject: [PATCH 18/97] Move offline log store Signed-off-by: Kevin Zhang --- go/{internal/feast => cmd/server/logging}/offline_log_store.go | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename go/{internal/feast => cmd/server/logging}/offline_log_store.go (100%) diff --git a/go/internal/feast/offline_log_store.go b/go/cmd/server/logging/offline_log_store.go similarity index 100% rename from go/internal/feast/offline_log_store.go rename to go/cmd/server/logging/offline_log_store.go From cb1da995ecff1019ced632656bf6d9bddd382d84 Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Wed, 30 Mar 2022 11:23:43 -0700 Subject: [PATCH 19/97] refactor Signed-off-by: Kevin Zhang --- go/cmd/server/logging/filelogstorage.go | 1 + go/cmd/server/logging/offline_log_store.go | 27 ------------------ go/cmd/server/logging/offlinelogstorage.go | 33 ++++++++++++++++++++++ 3 files changed, 34 insertions(+), 27 deletions(-) create mode 100644 go/cmd/server/logging/filelogstorage.go delete mode 100644 go/cmd/server/logging/offline_log_store.go create mode 100644 go/cmd/server/logging/offlinelogstorage.go diff --git a/go/cmd/server/logging/filelogstorage.go b/go/cmd/server/logging/filelogstorage.go new file mode 100644 index 00000000000..2b43acc6b6e --- /dev/null +++ b/go/cmd/server/logging/filelogstorage.go @@ -0,0 +1 @@ +package logging diff --git a/go/cmd/server/logging/offline_log_store.go b/go/cmd/server/logging/offline_log_store.go deleted file mode 100644 index 9f2c45b440f..00000000000 --- a/go/cmd/server/logging/offline_log_store.go +++ /dev/null @@ -1,27 +0,0 @@ -package feast - -// import "github.com/feast-dev/feast/go/cmd/server/logging" - -// type OfflineLogStorage interface { -// FlushToStorage(logging.MemoryBuffer) -// // Destruct must be call once user is done using OnlineStore -// // This is to comply with the Connector since we have to close the plugin -// } - -// func getOfflineStoreType(offlineStoreConfig map[string]interface{}) (string, bool) { -// if onlineStoreType, ok := offlineStoreConfig["type"]; !ok { -// // Assume file for case of no specified. -// return "file", true -// } else { -// result, ok := onlineStoreType.(string) -// return result, ok -// } -// } - -// func NewOfflineStore(config *RepoConfig) (OfflineLogStorage, error) { -// onlineStoreType, ok := getOfflineStoreType(config.OfflineStore) -// if !ok { -// onlineStore, err := NewSqliteOnlineStore(config.Project, config, config.OnlineStore) -// return onlineStore, err -// } -// } diff --git a/go/cmd/server/logging/offlinelogstorage.go b/go/cmd/server/logging/offlinelogstorage.go new file mode 100644 index 00000000000..f1575026782 --- /dev/null +++ b/go/cmd/server/logging/offlinelogstorage.go @@ -0,0 +1,33 @@ +package logging + +import ( + "errors" + + "github.com/feast-dev/feast/go/internal/feast" +) + +type OfflineLogStorage interface { + FlushToStorage(MemoryBuffer) + // Destruct must be call once user is done using OnlineStore + // This is to comply with the Connector since we have to close the plugin +} + +func getOfflineStoreType(offlineStoreConfig map[string]interface{}) (string, bool) { + if onlineStoreType, ok := offlineStoreConfig["type"]; !ok { + // Assume file for case of no specified. + return "file", true + } else { + result, ok := onlineStoreType.(string) + return result, ok + } +} + +func NewOfflineStore(config *feast.RepoConfig) (OfflineLogStorage, error) { + onlineStoreType, _ := getOfflineStoreType(config.OfflineStore) + if onlineStoreType == "file" { + offlineStore, err := NewFileOfflineStore(config.Project, config.OfflineStore) + return offlineStore, err + } else { + return nil, errors.New("No offline storage besides file is currently supported.") + } +} From 155fc77493357747d0d13993ad05e2e0fb6a2b1c Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Wed, 30 Mar 2022 12:19:21 -0700 Subject: [PATCH 20/97] Update logs Signed-off-by: Kevin Zhang --- go/cmd/server/logging/filelogstorage.go | 51 ++++++++++++++++++++ go/cmd/server/logging/filelogstorage_test.go | 1 + go/cmd/server/logging/offlinelogstorage.go | 4 +- 3 files changed, 54 insertions(+), 2 deletions(-) create mode 100644 go/cmd/server/logging/filelogstorage_test.go diff --git a/go/cmd/server/logging/filelogstorage.go b/go/cmd/server/logging/filelogstorage.go index 2b43acc6b6e..74967210949 100644 --- a/go/cmd/server/logging/filelogstorage.go +++ b/go/cmd/server/logging/filelogstorage.go @@ -1 +1,52 @@ package logging + +import ( + "time" + + "github.com/feast-dev/feast/go/protos/feast/serving" +) + +type FileLogStorage struct { + // Feast project name + project string + path string +} + +type ParquetLog struct { + EntityName string `parquet:"name=entityname, type=BYTE_ARRAY"` + FeatureNames []string `parquet:"name=featurenames, type=MAP, convertedtype=LIST, valuetype=BYTE_ARRAY, valueconvertedtype=UTF8"` + FeatureStatuses []bool `parquet:"name=featurestatuses, type=BOOLEAN, repetitiontype=REPEATED"` + EventTimestamps []int64 `parquet:"name=eventtimestamps, type=INT64, repetitiontype=REPEATED, convertedtype=TIMESTAMP_MILLIS"` +} + +func NewFileOfflineStore(project string, offlineStoreConfig map[string]interface{}) (*FileLogStorage, error) { + store := FileLogStorage{project: project} + return &store, nil +} + +func (f *FileLogStorage) FlushToStorage(m *MemoryBuffer) error { + if len(m.logs) == 0 { + return nil + } + for _, log := range m.logs { + numValues := len(log.FeatureValues) + statuses := make([]bool, numValues) + timestampsInMillis := make([]int64, numValues) + for idx := 0; idx < numValues; idx++ { + if log.FeatureStatuses[idx] == serving.FieldStatus_PRESENT { + statuses[idx] = true + } else { + statuses[idx] = false + } + ts := log.EventTimestamps[idx] + timestampsInMillis[idx] = ts.AsTime().UnixNano() / int64(time.Millisecond) + } + newParquetLog := ParquetLog{ + EntityName: log.EntityName, + FeatureNames: log.FeatureNames, + FeatureStatuses: statuses, + EventTimestamps: timestampsInMillis, + } + } + return nil +} diff --git a/go/cmd/server/logging/filelogstorage_test.go b/go/cmd/server/logging/filelogstorage_test.go new file mode 100644 index 00000000000..2b43acc6b6e --- /dev/null +++ b/go/cmd/server/logging/filelogstorage_test.go @@ -0,0 +1 @@ +package logging diff --git a/go/cmd/server/logging/offlinelogstorage.go b/go/cmd/server/logging/offlinelogstorage.go index f1575026782..422875d9a3e 100644 --- a/go/cmd/server/logging/offlinelogstorage.go +++ b/go/cmd/server/logging/offlinelogstorage.go @@ -7,7 +7,7 @@ import ( ) type OfflineLogStorage interface { - FlushToStorage(MemoryBuffer) + FlushToStorage(*MemoryBuffer) error // Destruct must be call once user is done using OnlineStore // This is to comply with the Connector since we have to close the plugin } @@ -28,6 +28,6 @@ func NewOfflineStore(config *feast.RepoConfig) (OfflineLogStorage, error) { offlineStore, err := NewFileOfflineStore(config.Project, config.OfflineStore) return offlineStore, err } else { - return nil, errors.New("No offline storage besides file is currently supported.") + return nil, errors.New("no offline storage besides file is currently supported") } } From 86f220889d0171c71711f516b8ff06e4e41191a1 Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Wed, 30 Mar 2022 12:30:33 -0700 Subject: [PATCH 21/97] Update log storage Signed-off-by: Kevin Zhang --- go/cmd/server/logging/filelogstorage.go | 31 +++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/go/cmd/server/logging/filelogstorage.go b/go/cmd/server/logging/filelogstorage.go index 74967210949..7d0546f02a3 100644 --- a/go/cmd/server/logging/filelogstorage.go +++ b/go/cmd/server/logging/filelogstorage.go @@ -1,9 +1,12 @@ package logging import ( + "fmt" + "os" "time" "github.com/feast-dev/feast/go/protos/feast/serving" + "github.com/xitongsys/parquet-go/writer" ) type FileLogStorage struct { @@ -24,10 +27,38 @@ func NewFileOfflineStore(project string, offlineStoreConfig map[string]interface return &store, nil } +func CreateOrOpenLogFile(absPath string) (*os.File, error) { + var _, err = os.Stat(absPath) + + // create file if not exists + if os.IsNotExist(err) { + var file, err = os.Create(absPath) + if err != nil { + return nil, err + } + return file, nil + } else { + var file, err = os.OpenFile(absPath, os.O_RDWR, 0644) + if err != nil { + return nil, err + } + return file, nil + } +} + func (f *FileLogStorage) FlushToStorage(m *MemoryBuffer) error { if len(m.logs) == 0 { return nil } + var err error + w, err := CreateOrOpenLogFile("output/flat.parquet") + if err != nil { + return fmt.Errorf("Can't create local file with error: %s", err) + } + pw, err := writer.NewParquetWriterFromWriter(w, new(ParquetLog), 4) + if err != nil { + return fmt.Errorf("Can't create parquet writer with error: %s", err) + } for _, log := range m.logs { numValues := len(log.FeatureValues) statuses := make([]bool, numValues) From 746dae9900c3ebe063acf59f5b118d2b270b2a45 Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Wed, 30 Mar 2022 13:48:31 -0700 Subject: [PATCH 22/97] WOrking state Signed-off-by: Kevin Zhang --- go/cmd/server/logging/filelogstorage.go | 40 ++++++++---- go/cmd/server/logging/filelogstorage_test.go | 64 ++++++++++++++++++++ 2 files changed, 93 insertions(+), 11 deletions(-) diff --git a/go/cmd/server/logging/filelogstorage.go b/go/cmd/server/logging/filelogstorage.go index 7d0546f02a3..d183c749f1f 100644 --- a/go/cmd/server/logging/filelogstorage.go +++ b/go/cmd/server/logging/filelogstorage.go @@ -2,7 +2,9 @@ package logging import ( "fmt" + "log" "os" + "path/filepath" "time" "github.com/feast-dev/feast/go/protos/feast/serving" @@ -18,12 +20,17 @@ type FileLogStorage struct { type ParquetLog struct { EntityName string `parquet:"name=entityname, type=BYTE_ARRAY"` FeatureNames []string `parquet:"name=featurenames, type=MAP, convertedtype=LIST, valuetype=BYTE_ARRAY, valueconvertedtype=UTF8"` - FeatureStatuses []bool `parquet:"name=featurestatuses, type=BOOLEAN, repetitiontype=REPEATED"` - EventTimestamps []int64 `parquet:"name=eventtimestamps, type=INT64, repetitiontype=REPEATED, convertedtype=TIMESTAMP_MILLIS"` + FeatureStatuses []bool `parquet:"name=featurestatuses, type=MAP, convertedtype=LIST, valuetype=BOOLEAN"` + EventTimestamps []int64 `parquet:"name=eventtimestamps, type=MAP, convertedtype=LIST, valuetype=INT64, valueconvertedtype=TIMESTAMP_MILLIS"` } func NewFileOfflineStore(project string, offlineStoreConfig map[string]interface{}) (*FileLogStorage, error) { store := FileLogStorage{project: project} + abs_path, err := filepath.Abs("log.parquet") + if err != nil { + return nil, err + } + store.path = abs_path return &store, nil } @@ -51,33 +58,44 @@ func (f *FileLogStorage) FlushToStorage(m *MemoryBuffer) error { return nil } var err error - w, err := CreateOrOpenLogFile("output/flat.parquet") + w, err := CreateOrOpenLogFile(f.path) if err != nil { - return fmt.Errorf("Can't create local file with error: %s", err) + return fmt.Errorf("can't create local file with error: %s", err) } pw, err := writer.NewParquetWriterFromWriter(w, new(ParquetLog), 4) if err != nil { - return fmt.Errorf("Can't create parquet writer with error: %s", err) + return fmt.Errorf("can't create parquet writer with error: %s", err) } - for _, log := range m.logs { - numValues := len(log.FeatureValues) + for _, newLog := range m.logs { + numValues := len(newLog.FeatureValues) statuses := make([]bool, numValues) timestampsInMillis := make([]int64, numValues) for idx := 0; idx < numValues; idx++ { - if log.FeatureStatuses[idx] == serving.FieldStatus_PRESENT { + if newLog.FeatureStatuses[idx] == serving.FieldStatus_PRESENT { statuses[idx] = true } else { statuses[idx] = false } - ts := log.EventTimestamps[idx] + ts := newLog.EventTimestamps[idx] timestampsInMillis[idx] = ts.AsTime().UnixNano() / int64(time.Millisecond) } + log.Println(statuses) + log.Println(timestampsInMillis) newParquetLog := ParquetLog{ - EntityName: log.EntityName, - FeatureNames: log.FeatureNames, + EntityName: newLog.EntityName, + FeatureNames: newLog.FeatureNames, FeatureStatuses: statuses, EventTimestamps: timestampsInMillis, } + if err = pw.Write(newParquetLog); err != nil { + log.Println("write error") + return err + } + } + if err = pw.WriteStop(); err != nil { + return err } + log.Println("Flushed Log to Parquet File Storage") + w.Close() return nil } diff --git a/go/cmd/server/logging/filelogstorage_test.go b/go/cmd/server/logging/filelogstorage_test.go index 2b43acc6b6e..62959b47f49 100644 --- a/go/cmd/server/logging/filelogstorage_test.go +++ b/go/cmd/server/logging/filelogstorage_test.go @@ -1 +1,65 @@ package logging + +import ( + "log" + "os" + "testing" + "time" + + "github.com/feast-dev/feast/go/protos/feast/serving" + "github.com/feast-dev/feast/go/protos/feast/types" + "github.com/stretchr/testify/assert" + "github.com/xitongsys/parquet-go-source/local" + "github.com/xitongsys/parquet-go/reader" + "google.golang.org/protobuf/types/known/timestamppb" +) + +func TestWriteToLogStorage(t *testing.T) { + offlineStoreConfig := map[string]interface{}{ + "path": "log.parquet", + } + fileStore, err := NewFileOfflineStore("test", offlineStoreConfig) + assert.Nil(t, err) + ts := timestamppb.New(time.Now()) + newLog := Log{ + FeatureNames: []string{"feature1", "feature2"}, + FeatureValues: []*types.Value{{Val: &types.Value_Int64Val{Int64Val: 1001}}, {Val: &types.Value_Int64Val{Int64Val: 1002}}}, + FeatureStatuses: []serving.FieldStatus{serving.FieldStatus_PRESENT}, + EventTimestamps: []*timestamppb.Timestamp{ts, ts}, + } + newLog2 := Log{ + FeatureNames: []string{"feature4", "feature5"}, + FeatureValues: []*types.Value{{Val: &types.Value_Int64Val{Int64Val: 1003}}, {Val: &types.Value_Int64Val{Int64Val: 1004}}}, + FeatureStatuses: []serving.FieldStatus{serving.FieldStatus_PRESENT}, + EventTimestamps: []*timestamppb.Timestamp{ts, ts}, + } + memoryBuffer := MemoryBuffer{ + logs: []*Log{&newLog, &newLog2}, + } + + err = fileStore.FlushToStorage(&memoryBuffer) + assert.Nil(t, err) + + ///read + fr, err := local.NewLocalFileReader("log.parquet") + assert.Nil(t, err) + + pr, err := reader.NewParquetReader(fr, new(ParquetLog), 4) + if err != nil { + log.Println("Can't create parquet reader", err) + return + } + num := int(pr.GetNumRows()) + assert.Equal(t, num, 2) + logs := make([]ParquetLog, 2) //read 10 rows + if err = pr.Read(&logs); err != nil { + log.Println("Read error", err) + } + log.Println(logs) + + pr.ReadStop() + fr.Close() + + err = os.Remove("log.parquet") + assert.Nil(t, err) +} From 514c2f647bbbffed2245855bf0dd3bd03177f0a3 Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Thu, 31 Mar 2022 11:12:32 -0700 Subject: [PATCH 23/97] Work Signed-off-by: Kevin Zhang --- go/cmd/server/logging/filelogstorage.go | 14 +++++++++++--- go/cmd/server/logging/filelogstorage_test.go | 14 +++++++++----- 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/go/cmd/server/logging/filelogstorage.go b/go/cmd/server/logging/filelogstorage.go index d183c749f1f..e77f03d9cb9 100644 --- a/go/cmd/server/logging/filelogstorage.go +++ b/go/cmd/server/logging/filelogstorage.go @@ -1,6 +1,7 @@ package logging import ( + "errors" "fmt" "log" "os" @@ -19,7 +20,9 @@ type FileLogStorage struct { type ParquetLog struct { EntityName string `parquet:"name=entityname, type=BYTE_ARRAY"` + EntityValue string `parquet:"name=entityvalue, type=BYTE_ARRAY"` FeatureNames []string `parquet:"name=featurenames, type=MAP, convertedtype=LIST, valuetype=BYTE_ARRAY, valueconvertedtype=UTF8"` + FeatureValues []string `parquet:"name=featurevalues, type=MAP, convertedtype=LIST, valuetype=BYTE_ARRAY, valueconvertedtype=UTF8"` FeatureStatuses []bool `parquet:"name=featurestatuses, type=MAP, convertedtype=LIST, valuetype=BOOLEAN"` EventTimestamps []int64 `parquet:"name=eventtimestamps, type=MAP, convertedtype=LIST, valuetype=INT64, valueconvertedtype=TIMESTAMP_MILLIS"` } @@ -66,10 +69,15 @@ func (f *FileLogStorage) FlushToStorage(m *MemoryBuffer) error { if err != nil { return fmt.Errorf("can't create parquet writer with error: %s", err) } + for _, newLog := range m.logs { numValues := len(newLog.FeatureValues) + if numValues != len(newLog.FeatureStatuses) || numValues != len(newLog.EventTimestamps) { + return errors.New("length of log arrays do not match") + } statuses := make([]bool, numValues) timestampsInMillis := make([]int64, numValues) + featureValues := make([]string, numValues) for idx := 0; idx < numValues; idx++ { if newLog.FeatureStatuses[idx] == serving.FieldStatus_PRESENT { statuses[idx] = true @@ -78,17 +86,17 @@ func (f *FileLogStorage) FlushToStorage(m *MemoryBuffer) error { } ts := newLog.EventTimestamps[idx] timestampsInMillis[idx] = ts.AsTime().UnixNano() / int64(time.Millisecond) + featureValues[idx] = newLog.FeatureValues[idx].String() } - log.Println(statuses) - log.Println(timestampsInMillis) newParquetLog := ParquetLog{ EntityName: newLog.EntityName, + EntityValue: newLog.EntityValue.String(), FeatureNames: newLog.FeatureNames, + FeatureValues: featureValues, FeatureStatuses: statuses, EventTimestamps: timestampsInMillis, } if err = pw.Write(newParquetLog); err != nil { - log.Println("write error") return err } } diff --git a/go/cmd/server/logging/filelogstorage_test.go b/go/cmd/server/logging/filelogstorage_test.go index 62959b47f49..f2e4c9e990c 100644 --- a/go/cmd/server/logging/filelogstorage_test.go +++ b/go/cmd/server/logging/filelogstorage_test.go @@ -22,15 +22,19 @@ func TestWriteToLogStorage(t *testing.T) { assert.Nil(t, err) ts := timestamppb.New(time.Now()) newLog := Log{ - FeatureNames: []string{"feature1", "feature2"}, - FeatureValues: []*types.Value{{Val: &types.Value_Int64Val{Int64Val: 1001}}, {Val: &types.Value_Int64Val{Int64Val: 1002}}}, - FeatureStatuses: []serving.FieldStatus{serving.FieldStatus_PRESENT}, + EntityName: "driver_id", + EntityValue: &types.Value{Val: &types.Value_Int64Val{Int64Val: 1001}}, + FeatureNames: []string{"conv_rate", "acc_rate"}, + FeatureValues: []*types.Value{{Val: &types.Value_FloatVal{FloatVal: 0.2}}, {Val: &types.Value_FloatVal{FloatVal: 0.5}}}, + FeatureStatuses: []serving.FieldStatus{serving.FieldStatus_PRESENT, serving.FieldStatus_PRESENT}, EventTimestamps: []*timestamppb.Timestamp{ts, ts}, } newLog2 := Log{ + EntityName: "driver_id", + EntityValue: &types.Value{Val: &types.Value_Int64Val{Int64Val: 1003}}, FeatureNames: []string{"feature4", "feature5"}, - FeatureValues: []*types.Value{{Val: &types.Value_Int64Val{Int64Val: 1003}}, {Val: &types.Value_Int64Val{Int64Val: 1004}}}, - FeatureStatuses: []serving.FieldStatus{serving.FieldStatus_PRESENT}, + FeatureValues: []*types.Value{{Val: &types.Value_FloatVal{FloatVal: 0.3}}, {Val: &types.Value_FloatVal{FloatVal: 0.8}}}, + FeatureStatuses: []serving.FieldStatus{serving.FieldStatus_PRESENT, serving.FieldStatus_PRESENT}, EventTimestamps: []*timestamppb.Timestamp{ts, ts}, } memoryBuffer := MemoryBuffer{ From 161d81d939dbe3c8475b08339679712f3201155f Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Thu, 31 Mar 2022 11:50:52 -0700 Subject: [PATCH 24/97] Add tests for filestorage Signed-off-by: Kevin Zhang --- go/cmd/server/logging/filelogstorage_test.go | 21 +++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/go/cmd/server/logging/filelogstorage_test.go b/go/cmd/server/logging/filelogstorage_test.go index f2e4c9e990c..621175392e1 100644 --- a/go/cmd/server/logging/filelogstorage_test.go +++ b/go/cmd/server/logging/filelogstorage_test.go @@ -3,6 +3,7 @@ package logging import ( "log" "os" + "reflect" "testing" "time" @@ -59,7 +60,25 @@ func TestWriteToLogStorage(t *testing.T) { if err = pr.Read(&logs); err != nil { log.Println("Read error", err) } - log.Println(logs) + + for i := 0; i < 2; i++ { + assert.Equal(t, logs[i].EntityName, memoryBuffer.logs[i].EntityName) + assert.Equal(t, logs[i].EntityValue, memoryBuffer.logs[i].EntityValue.String()) + assert.True(t, reflect.DeepEqual(logs[i].FeatureNames, memoryBuffer.logs[i].FeatureNames)) + numValues := len(memoryBuffer.logs[i].FeatureValues) + assert.Equal(t, numValues, len(logs[i].FeatureValues)) + assert.Equal(t, numValues, len(logs[i].EventTimestamps)) + assert.Equal(t, numValues, len(logs[i].FeatureStatuses)) + for idx := 0; idx < numValues; idx++ { + assert.Equal(t, logs[i].EventTimestamps[idx], memoryBuffer.logs[i].EventTimestamps[idx].AsTime().UnixNano()/int64(time.Millisecond)) + if memoryBuffer.logs[i].FeatureStatuses[idx] == serving.FieldStatus_PRESENT { + assert.True(t, logs[i].FeatureStatuses[idx]) + } else { + assert.False(t, logs[i].FeatureStatuses[idx]) + } + assert.Equal(t, logs[i].FeatureValues[idx], memoryBuffer.logs[i].FeatureValues[idx].String()) + } + } pr.ReadStop() fr.Close() From beec6b20279ccf0b2a8ba3fda7b185aebcbcdd83 Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Thu, 31 Mar 2022 13:01:14 -0700 Subject: [PATCH 25/97] Fix logging Signed-off-by: Kevin Zhang --- go/cmd/server/logging/logging.go | 54 ++++++++++++++-------- go/cmd/server/logging/logging_test.go | 11 +++-- go/cmd/server/logging/offlinelogstorage.go | 2 - go/cmd/server/main.go | 7 +-- go/cmd/server/server.go | 5 -- go/cmd/server/server_test.go | 13 ++++-- go/internal/feast/featurestore.go | 5 -- 7 files changed, 50 insertions(+), 47 deletions(-) diff --git a/go/cmd/server/logging/logging.go b/go/cmd/server/logging/logging.go index 8fc265d05b8..560d75033a7 100644 --- a/go/cmd/server/logging/logging.go +++ b/go/cmd/server/logging/logging.go @@ -1,6 +1,7 @@ package logging import ( + "errors" "fmt" "log" "time" @@ -31,25 +32,38 @@ type MemoryBuffer struct { } type LoggingService struct { - memoryBuffer *MemoryBuffer - logChannel chan *Log - fs *feast.FeatureStore + memoryBuffer *MemoryBuffer + logChannel chan *Log + fs *feast.FeatureStore + offlineLogStorage OfflineLogStorage + enableLogging bool } -func NewLoggingService(fs *feast.FeatureStore, logChannelCapacity int, startLogProcessing bool) *LoggingService { +func NewLoggingService(fs *feast.FeatureStore, logChannelCapacity int, enableLogging bool) (*LoggingService, error) { // start handler processes? loggingService := &LoggingService{ logChannel: make(chan *Log, logChannelCapacity), memoryBuffer: &MemoryBuffer{ logs: make([]*Log, 0), }, - fs: fs, + enableLogging: enableLogging, + fs: fs, + } + if !enableLogging || fs == nil { + loggingService.offlineLogStorage = nil + } else { + offlineLogStorage, err := NewOfflineStore(fs.GetRepoConfig()) + loggingService.offlineLogStorage = offlineLogStorage + + if err != nil { + return nil, err + } } // For testing purposes, so we can test timeouts. - if startLogProcessing { + if enableLogging { go loggingService.processLogs() } - return loggingService + return loggingService, nil } func (s *LoggingService) EmitLog(log *Log) error { @@ -63,6 +77,7 @@ func (s *LoggingService) EmitLog(log *Log) error { func (s *LoggingService) processLogs() { // start a periodic flush + // TODO(kevjumba): set param so users can configure this ticker := time.NewTicker(100 * time.Millisecond) defer ticker.Stop() for { @@ -76,21 +91,20 @@ func (s *LoggingService) processLogs() { } } -func (s *LoggingService) flushLogsToOfflineStorage(t time.Time) { - //offlineStore := fs.config.OfflineStore["type"] - // switch offlineStore{ - // case "file": - // // call python?? - // case "snowflake": - // - // } - //Do different row level manipulations and add to offline store +func (s *LoggingService) flushLogsToOfflineStorage(t time.Time) error { log.Printf("Flushing buffer to offline storage with channel length: %d\n at time: "+t.String(), len(s.memoryBuffer.logs)) - if s.fs.GetRepoConfig().OfflineStore["type"] == "file" { - //file_path := s.fs.GetRepoConfig().RepoPath - + if !s.enableLogging { + return nil + } + offlineStoreType, ok := getOfflineStoreType(s.fs.GetRepoConfig().OfflineStore) + if !ok { + return fmt.Errorf("could not get offline storage type for config: %s", s.fs.GetRepoConfig().OfflineStore) + } + if offlineStoreType == "file" { + s.offlineLogStorage.FlushToStorage(s.memoryBuffer) } else { // Currently don't support any other offline flushing. - return + return errors.New("currently only file type is supported for offline log storage") } + return nil } diff --git a/go/cmd/server/logging/logging_test.go b/go/cmd/server/logging/logging_test.go index 46d44a9c101..a6e50d849fa 100644 --- a/go/cmd/server/logging/logging_test.go +++ b/go/cmd/server/logging/logging_test.go @@ -12,7 +12,8 @@ import ( func TestLoggingChannelToMemoryBuffer(t *testing.T) { //Feature store is still not checked in so we can't create one. - loggingService := NewLoggingService(nil, 10, true) + loggingService, err := NewLoggingService(nil, 10, true) + assert.Nil(t, err) assert.Empty(t, loggingService.memoryBuffer.logs) ts := timestamppb.New(time.Now()) newLog := Log{ @@ -20,7 +21,8 @@ func TestLoggingChannelToMemoryBuffer(t *testing.T) { FeatureStatuses: []serving.FieldStatus{serving.FieldStatus_PRESENT}, EventTimestamps: []*timestamppb.Timestamp{ts}, } - err := loggingService.EmitLog(&newLog) + err = loggingService.EmitLog(&newLog) + assert.Nil(t, err) // Wait for memory buffer flush time.Sleep(20 * time.Millisecond) assert.Len(t, loggingService.memoryBuffer.logs, 1) @@ -32,7 +34,8 @@ func TestLoggingChannelToMemoryBuffer(t *testing.T) { func TestLoggingChannelTimeout(t *testing.T) { //Feature store is still not checked in so we can't create one. - loggingService := NewLoggingService(nil, 1, false) + loggingService, err := NewLoggingService(nil, 1, false) + assert.Nil(t, err) assert.Empty(t, loggingService.memoryBuffer.logs) ts := timestamppb.New(time.Now()) newLog := Log{ @@ -50,7 +53,7 @@ func TestLoggingChannelTimeout(t *testing.T) { FeatureStatuses: []serving.FieldStatus{serving.FieldStatus_PRESENT}, EventTimestamps: []*timestamppb.Timestamp{newTs, newTs}, } - err := loggingService.EmitLog(&newLog2) + err = loggingService.EmitLog(&newLog2) time.Sleep(20 * time.Millisecond) assert.NotNil(t, err) } diff --git a/go/cmd/server/logging/offlinelogstorage.go b/go/cmd/server/logging/offlinelogstorage.go index 422875d9a3e..6e3112b90c2 100644 --- a/go/cmd/server/logging/offlinelogstorage.go +++ b/go/cmd/server/logging/offlinelogstorage.go @@ -8,8 +8,6 @@ import ( type OfflineLogStorage interface { FlushToStorage(*MemoryBuffer) error - // Destruct must be call once user is done using OnlineStore - // This is to comply with the Connector since we have to close the plugin } func getOfflineStoreType(offlineStoreConfig map[string]interface{}) (string, bool) { diff --git a/go/cmd/server/main.go b/go/cmd/server/main.go index 9404c998412..d5ee473362f 100644 --- a/go/cmd/server/main.go +++ b/go/cmd/server/main.go @@ -44,13 +44,8 @@ func main() { } log.Println("Initializing feature store...") -<<<<<<< HEAD fs, err := feast.NewFeatureStore(repoConfig, nil) - loggingService := NewLoggingService(fs, 1000, true) -======= - fs, err := feast.NewFeatureStore(repoConfig) - loggingService := logging.NewLoggingService(fs, 1000, true) ->>>>>>> 118a377d (Working state) + loggingService, err := logging.NewLoggingService(fs, 1000, true) if err != nil { log.Fatalln(err) } diff --git a/go/cmd/server/server.go b/go/cmd/server/server.go index 88e36608ca6..acb5ff1b62f 100644 --- a/go/cmd/server/server.go +++ b/go/cmd/server/server.go @@ -52,10 +52,6 @@ func (s *servingServiceServer) GetOnlineFeatures(ctx context.Context, request *s FeatureNames: &serving.FeatureList{Val: make([]string, 0)}, }, } -<<<<<<< HEAD - - for _, vector := range featureVectors { -======= // Entities are currently part of the features as a value entityNames := make([]string, 0) entityValues := make([]*prototypes.Value, 0) @@ -77,7 +73,6 @@ func (s *servingServiceServer) GetOnlineFeatures(ctx context.Context, request *s } featureNames := make([]string, len(featureVectors)) for idx, vector := range featureVectors { ->>>>>>> 118a377d (Working state) resp.Metadata.FeatureNames.Val = append(resp.Metadata.FeatureNames.Val, vector.Name) featureNames[idx] = vector.Name values, err := types.ArrowValuesToProtoValues(vector.Values) diff --git a/go/cmd/server/server_test.go b/go/cmd/server/server_test.go index 354f6a60262..5d69a1339e9 100644 --- a/go/cmd/server/server_test.go +++ b/go/cmd/server/server_test.go @@ -35,7 +35,7 @@ func getRepoPath(basePath string) string { } // Starts a new grpc server, registers the serving service and returns a client. -func getClient(ctx context.Context, basePath string) (serving.ServingServiceClient, func()) { +func getClient(ctx context.Context, basePath string, enableLogging bool) (serving.ServingServiceClient, func()) { buffer := 1024 * 1024 listener := bufconn.Listen(buffer) @@ -48,7 +48,10 @@ func getClient(ctx context.Context, basePath string) (serving.ServingServiceClie if err != nil { panic(err) } - loggingService := logging.NewLoggingService(fs, 1000, true) + loggingService, err := logging.NewLoggingService(fs, 1000, enableLogging) + if err != nil { + panic(err) + } servingServiceServer := newServingServiceServer(fs, loggingService) serving.RegisterServingServiceServer(server, servingServiceServer) @@ -79,7 +82,7 @@ func TestGetFeastServingInfo(t *testing.T) { err := test.SetupFeatureRepo(dir) assert.Nil(t, err) defer test.CleanUpRepo(dir) - client, closer := getClient(ctx, dir) + client, closer := getClient(ctx, dir, false) defer closer() response, err := client.GetFeastServingInfo(ctx, &serving.GetFeastServingInfoRequest{}) assert.Nil(t, err) @@ -93,7 +96,7 @@ func TestGetOnlineFeaturesSqlite(t *testing.T) { err := test.SetupFeatureRepo(dir) assert.Nil(t, err) defer test.CleanUpRepo(dir) - client, closer := getClient(ctx, dir) + client, closer := getClient(ctx, dir, false) defer closer() entities := make(map[string]*types.RepeatedValue) entities["driver_id"] = &types.RepeatedValue{ @@ -152,7 +155,7 @@ func TestGetOnlineFeaturesSqliteWithLogging(t *testing.T) { err := test.SetupFeatureRepo(dir) assert.Nil(t, err) defer test.CleanUpRepo(dir) - client, closer := getClient(ctx, dir) + client, closer := getClient(ctx, dir, true) defer closer() entities := make(map[string]*types.RepeatedValue) entities["driver_id"] = &types.RepeatedValue{ diff --git a/go/internal/feast/featurestore.go b/go/internal/feast/featurestore.go index 4cefa9fe2fa..e05f2c61e16 100644 --- a/go/internal/feast/featurestore.go +++ b/go/internal/feast/featurestore.go @@ -32,14 +32,9 @@ type Features struct { // NewFeatureStore constructs a feature store fat client using the // repo config (contents of feature_store.yaml converted to JSON map). -<<<<<<< HEAD func NewFeatureStore(config *registry.RepoConfig, callback transformation.TransformationCallback) (*FeatureStore, error) { onlineStore, err := onlinestore.NewOnlineStore(config) -======= -func NewFeatureStore(config *RepoConfig) (*FeatureStore, error) { - onlineStore, err := NewOnlineStore(config) // offlineStore, err := NewOfflineStore(config) ->>>>>>> 118a377d (Working state) if err != nil { return nil, err } From 2a1d2ec59bc166d8d3b65fc23cff20a1660d2392 Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Thu, 31 Mar 2022 14:31:41 -0700 Subject: [PATCH 26/97] Add more tests Signed-off-by: Kevin Zhang --- go/cmd/server/logging/filelogstorage.go | 12 ++- go/cmd/server/logging/filelogstorage_test.go | 2 +- go/cmd/server/logging/logging.go | 1 + go/cmd/server/logging/logging_test.go | 43 +++++----- go/cmd/server/server_test.go | 87 +++++++++++++++++++- 5 files changed, 120 insertions(+), 25 deletions(-) diff --git a/go/cmd/server/logging/filelogstorage.go b/go/cmd/server/logging/filelogstorage.go index e77f03d9cb9..352ff979f23 100644 --- a/go/cmd/server/logging/filelogstorage.go +++ b/go/cmd/server/logging/filelogstorage.go @@ -29,7 +29,17 @@ type ParquetLog struct { func NewFileOfflineStore(project string, offlineStoreConfig map[string]interface{}) (*FileLogStorage, error) { store := FileLogStorage{project: project} - abs_path, err := filepath.Abs("log.parquet") + var abs_path string + var err error + if val, ok := offlineStoreConfig["path"]; !ok { + abs_path, err = filepath.Abs("log.parquet") + } else { + result, ok := val.(string) + if !ok { + return nil, errors.New("cannot convert offlinestore path to string") + } + abs_path, err = filepath.Abs(filepath.Join(result, "log.parquet")) + } if err != nil { return nil, err } diff --git a/go/cmd/server/logging/filelogstorage_test.go b/go/cmd/server/logging/filelogstorage_test.go index 621175392e1..15f51415420 100644 --- a/go/cmd/server/logging/filelogstorage_test.go +++ b/go/cmd/server/logging/filelogstorage_test.go @@ -17,7 +17,7 @@ import ( func TestWriteToLogStorage(t *testing.T) { offlineStoreConfig := map[string]interface{}{ - "path": "log.parquet", + "path": ".", } fileStore, err := NewFileOfflineStore("test", offlineStoreConfig) assert.Nil(t, err) diff --git a/go/cmd/server/logging/logging.go b/go/cmd/server/logging/logging.go index 560d75033a7..af016ce41ee 100644 --- a/go/cmd/server/logging/logging.go +++ b/go/cmd/server/logging/logging.go @@ -102,6 +102,7 @@ func (s *LoggingService) flushLogsToOfflineStorage(t time.Time) error { } if offlineStoreType == "file" { s.offlineLogStorage.FlushToStorage(s.memoryBuffer) + s.memoryBuffer.logs = s.memoryBuffer.logs[:0] } else { // Currently don't support any other offline flushing. return errors.New("currently only file type is supported for offline log storage") diff --git a/go/cmd/server/logging/logging_test.go b/go/cmd/server/logging/logging_test.go index a6e50d849fa..6e593a56269 100644 --- a/go/cmd/server/logging/logging_test.go +++ b/go/cmd/server/logging/logging_test.go @@ -1,7 +1,6 @@ package logging import ( - "reflect" "testing" "time" @@ -10,27 +9,27 @@ import ( "google.golang.org/protobuf/types/known/timestamppb" ) -func TestLoggingChannelToMemoryBuffer(t *testing.T) { - //Feature store is still not checked in so we can't create one. - loggingService, err := NewLoggingService(nil, 10, true) - assert.Nil(t, err) - assert.Empty(t, loggingService.memoryBuffer.logs) - ts := timestamppb.New(time.Now()) - newLog := Log{ - FeatureNames: []string{"feature1", "feature2"}, - FeatureStatuses: []serving.FieldStatus{serving.FieldStatus_PRESENT}, - EventTimestamps: []*timestamppb.Timestamp{ts}, - } - err = loggingService.EmitLog(&newLog) - assert.Nil(t, err) - // Wait for memory buffer flush - time.Sleep(20 * time.Millisecond) - assert.Len(t, loggingService.memoryBuffer.logs, 1) - assert.Len(t, loggingService.logChannel, 0) - assert.True(t, reflect.DeepEqual(loggingService.memoryBuffer.logs[0].FeatureNames, []string{"feature1", "feature2"})) - assert.True(t, reflect.DeepEqual(loggingService.memoryBuffer.logs[0].FeatureStatuses, []serving.FieldStatus{serving.FieldStatus_PRESENT})) - assert.Nil(t, err) -} +// func TestLoggingChannelToMemoryBuffer(t *testing.T) { +// //Feature store is still not checked in so we can't create one. +// loggingService, err := NewLoggingService(nil, 10, false) +// assert.Nil(t, err) +// assert.Empty(t, loggingService.memoryBuffer.logs) +// ts := timestamppb.New(time.Now()) +// newLog := Log{ +// FeatureNames: []string{"feature1", "feature2"}, +// FeatureStatuses: []serving.FieldStatus{serving.FieldStatus_PRESENT}, +// EventTimestamps: []*timestamppb.Timestamp{ts}, +// } +// err = loggingService.EmitLog(&newLog) +// assert.Nil(t, err) +// // Wait for memory buffer flush +// time.Sleep(20 * time.Millisecond) +// assert.Len(t, loggingService.memoryBuffer.logs, 1) +// assert.Len(t, loggingService.logChannel, 0) +// assert.True(t, reflect.DeepEqual(loggingService.memoryBuffer.logs[0].FeatureNames, []string{"feature1", "feature2"})) +// assert.True(t, reflect.DeepEqual(loggingService.memoryBuffer.logs[0].FeatureStatuses, []serving.FieldStatus{serving.FieldStatus_PRESENT})) +// assert.Nil(t, err) +// } func TestLoggingChannelTimeout(t *testing.T) { //Feature store is still not checked in so we can't create one. diff --git a/go/cmd/server/server_test.go b/go/cmd/server/server_test.go index 5d69a1339e9..2be2cd696fd 100644 --- a/go/cmd/server/server_test.go +++ b/go/cmd/server/server_test.go @@ -3,7 +3,9 @@ package main import ( "context" "github.com/feast-dev/feast/go/internal/feast/registry" + "log" "net" + "os" "path/filepath" "reflect" "runtime" @@ -16,6 +18,8 @@ import ( "github.com/feast-dev/feast/go/protos/feast/serving" "github.com/feast-dev/feast/go/protos/feast/types" "github.com/stretchr/testify/assert" + "github.com/xitongsys/parquet-go-source/local" + "github.com/xitongsys/parquet-go/reader" "google.golang.org/grpc" "google.golang.org/grpc/test/bufconn" ) @@ -41,6 +45,18 @@ func getClient(ctx context.Context, basePath string, enableLogging bool) (servin server := grpc.NewServer() config, err := registry.NewRepoConfigFromFile(getRepoPath(basePath)) + + //TODO(kevjumba): either add this officially or talk in design review about what the correct solution to this is. + if enableLogging { + if config.OfflineStore == nil { + config.OfflineStore = map[string]interface{}{ + "path": ".", + } + } else { + config.OfflineStore["path"] = "." + } + } + if err != nil { panic(err) } @@ -165,10 +181,18 @@ func TestGetOnlineFeaturesSqliteWithLogging(t *testing.T) { {Val: &types.Value_Int64Val{Int64Val: 1005}}, }, } + featureNames := []string{"driver_hourly_stats:conv_rate", "driver_hourly_stats:acc_rate", "driver_hourly_stats:avg_daily_trips"} + expectedEntityValuesResp := []*types.Value{ + {Val: &types.Value_Int64Val{Int64Val: 1001}}, + {Val: &types.Value_Int64Val{Int64Val: 1003}}, + {Val: &types.Value_Int64Val{Int64Val: 1005}}, + } + expectedFeatureNamesResp := []string{"conv_rate", "acc_rate", "avg_daily_trips"} + request := &serving.GetOnlineFeaturesRequest{ Kind: &serving.GetOnlineFeaturesRequest_Features{ Features: &serving.FeatureList{ - Val: []string{"driver_hourly_stats:conv_rate", "driver_hourly_stats:acc_rate", "driver_hourly_stats:avg_daily_trips"}, + Val: featureNames, }, }, Entities: entities, @@ -176,5 +200,66 @@ func TestGetOnlineFeaturesSqliteWithLogging(t *testing.T) { response, err := client.GetOnlineFeatures(ctx, request) assert.Nil(t, err) assert.NotNil(t, response) + // Wait for logger to flush. + // TODO(Change this when we add param for flush duration) + time.Sleep(200 * time.Millisecond) + expectedLogValues, expectedLogStatuses, expectedLogMillis := GetExpectedLogRows(featureNames, response.Results) + ///read + fr, err := local.NewLocalFileReader("log.parquet") + assert.Nil(t, err) + + pr, err := reader.NewParquetReader(fr, new(logging.ParquetLog), 4) + if err != nil { + log.Println("Can't create parquet reader", err) + return + } + + num := int(pr.GetNumRows()) + assert.Equal(t, num, 3) + logs := make([]logging.ParquetLog, 3) //read 10 rows + err = pr.Read(&logs) + assert.Nil(t, err) + for i := 0; i < 3; i++ { + assert.Equal(t, logs[i].EntityName, "driver_id") + assert.Equal(t, logs[i].EntityValue, expectedEntityValuesResp[i].String()) + assert.True(t, reflect.DeepEqual(logs[i].FeatureNames, expectedFeatureNamesResp)) + numValues := len(expectedFeatureNamesResp) + assert.Equal(t, numValues, len(logs[i].FeatureValues)) + assert.Equal(t, numValues, len(logs[i].EventTimestamps)) + assert.Equal(t, numValues, len(logs[i].FeatureStatuses)) + assert.True(t, reflect.DeepEqual(logs[i].FeatureValues, expectedLogValues[i])) + assert.True(t, reflect.DeepEqual(logs[i].FeatureStatuses, expectedLogStatuses[i])) + assert.True(t, reflect.DeepEqual(logs[i].EventTimestamps, expectedLogMillis[i])) + } + + pr.ReadStop() + fr.Close() + + err = os.Remove("log.parquet") + assert.Nil(t, err) +} + +func GetExpectedLogRows(featureNames []string, results []*serving.GetOnlineFeaturesResponse_FeatureVector) ([][]string, [][]bool, [][]int64) { + numFeatures := len(featureNames) + numRows := len(results[0].Values) + featureValueLogRows := make([][]string, numRows) + featureStatusLogRows := make([][]bool, numRows) + eventTimestampLogRows := make([][]int64, numRows) + + for row_idx := 0; row_idx < numRows; row_idx++ { + featureValueLogRows[row_idx] = make([]string, numFeatures) + featureStatusLogRows[row_idx] = make([]bool, numFeatures) + eventTimestampLogRows[row_idx] = make([]int64, numFeatures) + for idx := 1; idx < len(results); idx++ { + featureValueLogRows[row_idx][idx-1] = results[idx].Values[row_idx].String() + if results[idx].Statuses[row_idx] == serving.FieldStatus_PRESENT { + featureStatusLogRows[row_idx][idx-1] = true + } else { + featureStatusLogRows[row_idx][idx-1] = false + } + eventTimestampLogRows[row_idx][idx-1] = results[idx].EventTimestamps[row_idx].AsTime().UnixNano() / int64(time.Millisecond) + } + } + return featureValueLogRows, featureStatusLogRows, eventTimestampLogRows } From 9446e05225d23bb6a1fc4820c8ef4a9733739746 Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Thu, 31 Mar 2022 14:39:32 -0700 Subject: [PATCH 27/97] Fix Signed-off-by: Kevin Zhang --- go/cmd/server/logging/logging.go | 1 + 1 file changed, 1 insertion(+) diff --git a/go/cmd/server/logging/logging.go b/go/cmd/server/logging/logging.go index af016ce41ee..c80b9b7c73e 100644 --- a/go/cmd/server/logging/logging.go +++ b/go/cmd/server/logging/logging.go @@ -102,6 +102,7 @@ func (s *LoggingService) flushLogsToOfflineStorage(t time.Time) error { } if offlineStoreType == "file" { s.offlineLogStorage.FlushToStorage(s.memoryBuffer) + //Clean memory buffer s.memoryBuffer.logs = s.memoryBuffer.logs[:0] } else { // Currently don't support any other offline flushing. From 31e0de84b43c148467ba4f53b0655908aa5b31a3 Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Thu, 31 Mar 2022 14:48:39 -0700 Subject: [PATCH 28/97] Fix Signed-off-by: Kevin Zhang --- go/cmd/server/logging/logging_test.go | 24 +----------------------- 1 file changed, 1 insertion(+), 23 deletions(-) diff --git a/go/cmd/server/logging/logging_test.go b/go/cmd/server/logging/logging_test.go index 6e593a56269..663d11cf97a 100644 --- a/go/cmd/server/logging/logging_test.go +++ b/go/cmd/server/logging/logging_test.go @@ -9,30 +9,7 @@ import ( "google.golang.org/protobuf/types/known/timestamppb" ) -// func TestLoggingChannelToMemoryBuffer(t *testing.T) { -// //Feature store is still not checked in so we can't create one. -// loggingService, err := NewLoggingService(nil, 10, false) -// assert.Nil(t, err) -// assert.Empty(t, loggingService.memoryBuffer.logs) -// ts := timestamppb.New(time.Now()) -// newLog := Log{ -// FeatureNames: []string{"feature1", "feature2"}, -// FeatureStatuses: []serving.FieldStatus{serving.FieldStatus_PRESENT}, -// EventTimestamps: []*timestamppb.Timestamp{ts}, -// } -// err = loggingService.EmitLog(&newLog) -// assert.Nil(t, err) -// // Wait for memory buffer flush -// time.Sleep(20 * time.Millisecond) -// assert.Len(t, loggingService.memoryBuffer.logs, 1) -// assert.Len(t, loggingService.logChannel, 0) -// assert.True(t, reflect.DeepEqual(loggingService.memoryBuffer.logs[0].FeatureNames, []string{"feature1", "feature2"})) -// assert.True(t, reflect.DeepEqual(loggingService.memoryBuffer.logs[0].FeatureStatuses, []serving.FieldStatus{serving.FieldStatus_PRESENT})) -// assert.Nil(t, err) -// } - func TestLoggingChannelTimeout(t *testing.T) { - //Feature store is still not checked in so we can't create one. loggingService, err := NewLoggingService(nil, 1, false) assert.Nil(t, err) assert.Empty(t, loggingService.memoryBuffer.logs) @@ -53,6 +30,7 @@ func TestLoggingChannelTimeout(t *testing.T) { EventTimestamps: []*timestamppb.Timestamp{newTs, newTs}, } err = loggingService.EmitLog(&newLog2) + // The channel times out and doesn't hang. time.Sleep(20 * time.Millisecond) assert.NotNil(t, err) } From 137d700a758a70cf228a0de7970b198f6db358f4 Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Thu, 31 Mar 2022 14:56:02 -0700 Subject: [PATCH 29/97] Clean up Signed-off-by: Kevin Zhang --- go/cmd/server/logging/logging.go | 6 ++---- go/cmd/server/main.go | 8 ++++++-- go/cmd/server/server_test.go | 5 +++-- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/go/cmd/server/logging/logging.go b/go/cmd/server/logging/logging.go index c80b9b7c73e..bf1c3c4b2b5 100644 --- a/go/cmd/server/logging/logging.go +++ b/go/cmd/server/logging/logging.go @@ -58,9 +58,7 @@ func NewLoggingService(fs *feast.FeatureStore, logChannelCapacity int, enableLog if err != nil { return nil, err } - } - // For testing purposes, so we can test timeouts. - if enableLogging { + // Start goroutine to process logs go loggingService.processLogs() } return loggingService, nil @@ -77,7 +75,7 @@ func (s *LoggingService) EmitLog(log *Log) error { func (s *LoggingService) processLogs() { // start a periodic flush - // TODO(kevjumba): set param so users can configure this + // TODO(kevjumba): set param so users can configure flushing duration ticker := time.NewTicker(100 * time.Millisecond) defer ticker.Stop() for { diff --git a/go/cmd/server/main.go b/go/cmd/server/main.go index d5ee473362f..34861addae4 100644 --- a/go/cmd/server/main.go +++ b/go/cmd/server/main.go @@ -44,8 +44,12 @@ func main() { } log.Println("Initializing feature store...") - fs, err := feast.NewFeatureStore(repoConfig, nil) - loggingService, err := logging.NewLoggingService(fs, 1000, true) + fs, err := feast.NewFeatureStore(repoConfig) + if err != nil { + log.Fatalln(err) + } + // Disable logging for now + loggingService, err := logging.NewLoggingService(fs, 1000, false) if err != nil { log.Fatalln(err) } diff --git a/go/cmd/server/server_test.go b/go/cmd/server/server_test.go index 2be2cd696fd..20f64cc8220 100644 --- a/go/cmd/server/server_test.go +++ b/go/cmd/server/server_test.go @@ -46,7 +46,8 @@ func getClient(ctx context.Context, basePath string, enableLogging bool) (servin server := grpc.NewServer() config, err := registry.NewRepoConfigFromFile(getRepoPath(basePath)) - //TODO(kevjumba): either add this officially or talk in design review about what the correct solution to this is. + //TODO(kevjumba): either add this officially or talk in design review about what the correct solution for what do with path. + // Currently in python we use the path in FileSource but it is not specified in configuration unless it is using file_url? if enableLogging { if config.OfflineStore == nil { config.OfflineStore = map[string]interface{}{ @@ -204,7 +205,7 @@ func TestGetOnlineFeaturesSqliteWithLogging(t *testing.T) { // TODO(Change this when we add param for flush duration) time.Sleep(200 * time.Millisecond) expectedLogValues, expectedLogStatuses, expectedLogMillis := GetExpectedLogRows(featureNames, response.Results) - ///read + // read from parquet log file fr, err := local.NewLocalFileReader("log.parquet") assert.Nil(t, err) From ee847cd59c0f38aedfb875744646d833ea4c9777 Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Thu, 31 Mar 2022 14:57:40 -0700 Subject: [PATCH 30/97] Update error Signed-off-by: Kevin Zhang --- go/internal/test/go_integration_test_utils.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/go/internal/test/go_integration_test_utils.go b/go/internal/test/go_integration_test_utils.go index 154bf36ff86..8e8b0f9a0cf 100644 --- a/go/internal/test/go_integration_test_utils.go +++ b/go/internal/test/go_integration_test_utils.go @@ -124,8 +124,7 @@ func CleanUpRepo(basePath string) error { } err = os.RemoveAll(feature_repo_path) if err != nil { - log.Fatalf("Couldn't remove feature repo path. %s", err) - return err + return fmt.Errorf("couldn't remove feature repo path %s", err) } return nil } From ce12eeee0331c4d19983558005cb98bdf747f309 Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Fri, 1 Apr 2022 12:14:58 -0700 Subject: [PATCH 31/97] semi working state Signed-off-by: Kevin Zhang --- go.mod | 6 +++ go.sum | 5 +++ go/cmd/server/logging/logging.go | 38 +++++++++++++++- go/cmd/server/server.go | 15 +++++++ go/internal/feast/basefeatureview.go | 43 +++++++++++++++++++ go/internal/feast/featurestore.go | 6 --- go/internal/feast/featureviewprojection.go | 41 ++++++++++++++++++ .../online_store/test_universal_online.py | 1 + 8 files changed, 148 insertions(+), 7 deletions(-) create mode 100644 go/internal/feast/basefeatureview.go create mode 100644 go/internal/feast/featureviewprojection.go diff --git a/go.mod b/go.mod index ad8f81ceeee..319605eda01 100644 --- a/go.mod +++ b/go.mod @@ -27,6 +27,7 @@ require ( github.com/golang/snappy v0.0.4 // indirect github.com/gonuts/commander v0.1.0 // indirect github.com/gonuts/flag v0.1.0 // indirect +<<<<<<< HEAD github.com/google/flatbuffers v2.0.5+incompatible // indirect github.com/klauspost/asmfmt v1.3.1 // indirect github.com/klauspost/compress v1.15.1 // indirect @@ -41,15 +42,20 @@ require ( golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3 // indirect golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd // indirect golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27 // indirect +======= +>>>>>>> 8a7ccbe5 (semi working state) github.com/google/flatbuffers v2.0.0+incompatible // indirect github.com/google/go-cmp v0.5.7 // indirect github.com/klauspost/compress v1.15.1 // indirect github.com/pierrec/lz4/v4 v4.1.9 // indirect + github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect golang.org/x/exp v0.0.0-20211028214138-64b4c8e87d1a // indirect + golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3 // indirect golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f // indirect golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359 // indirect golang.org/x/text v0.3.7 // indirect + golang.org/x/tools v0.1.10 // indirect golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect google.golang.org/genproto v0.0.0-20220126215142-9970aeb2e350 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect diff --git a/go.sum b/go.sum index a75bd9d96fd..b17f01aa905 100644 --- a/go.sum +++ b/go.sum @@ -145,7 +145,9 @@ github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiu github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/gonuts/commander v0.1.0 h1:EcDTiVw9oAVORFjQOEOuHQqcl6OXMyTgELocTq6zJ0I= github.com/gonuts/commander v0.1.0/go.mod h1:qkb5mSlcWodYgo7vs8ulLnXhfinhZsZcm6+H/z1JjgY= +github.com/gonuts/flag v0.1.0 h1:fqMv/MZ+oNGu0i9gp0/IQ/ZaPIDoAZBOBaJoV7viCWM= github.com/gonuts/flag v0.1.0/go.mod h1:ZTmTGtrSPejTo/SRNhCqwLTmiAgyBdCkLYhHrAoBdz4= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= @@ -296,6 +298,7 @@ github.com/pierrec/lz4/v4 v4.1.12 h1:44l88ehTZAUGW4VlO1QC4zkilL99M6Y9MXNwEs0uzP8 github.com/pierrec/lz4/v4 v4.1.12/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/profile v1.2.1/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -428,6 +431,8 @@ golang.org/x/mod v0.6.0-dev.0.20211013180041-c96bc1413d57 h1:LQmS1nU0twXLA96Kt7U golang.org/x/mod v0.6.0-dev.0.20211013180041-c96bc1413d57/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.5.1-0.20210830214625-1b1db11ec8f4/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= +golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3 h1:kQgndtyPBW/JIYERgdxfwMYh3AVStj88WQTlNDi2a+o= +golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= diff --git a/go/cmd/server/logging/logging.go b/go/cmd/server/logging/logging.go index bf1c3c4b2b5..85c1c46ccea 100644 --- a/go/cmd/server/logging/logging.go +++ b/go/cmd/server/logging/logging.go @@ -6,6 +6,7 @@ import ( "log" "time" + "github.com/apache/arrow/go/arrow/array" "github.com/feast-dev/feast/go/internal/feast" "github.com/feast-dev/feast/go/protos/feast/serving" "github.com/feast-dev/feast/go/protos/feast/types" @@ -27,8 +28,15 @@ type Log struct { RequestContext map[string]*types.RepeatedValue } +// driver_id, +// 1003, 1004 +// [acc rate conv rate avg_daily_trips] +// [entityvalues, acc_rate conv_rate avg_daily_trips, acc_ratestatus, conv_rate_status] +// [entityvalues, entity value] + type MemoryBuffer struct { - logs []*Log + featureService *feast.FeatureService + logs []*Log } type LoggingService struct { @@ -99,6 +107,7 @@ func (s *LoggingService) flushLogsToOfflineStorage(t time.Time) error { return fmt.Errorf("could not get offline storage type for config: %s", s.fs.GetRepoConfig().OfflineStore) } if offlineStoreType == "file" { + s.offlineLogStorage.FlushToStorage(s.memoryBuffer) //Clean memory buffer s.memoryBuffer.logs = s.memoryBuffer.logs[:0] @@ -108,3 +117,30 @@ func (s *LoggingService) flushLogsToOfflineStorage(t time.Time) error { } return nil } + +func (s *LoggingService) getLogInArrowTable(memoryBuffer *MemoryBuffer) (*array.Table, error) { + // input memoryBuffer -> featureColumns + // map[string]*type.Value + + // fields := make([]*arrow.Field, 0) + // columns := make([]array.Interface, 0) + // for idx, feature := range featureService.features { + // feature.Name + + // []*proto.Value = columnNameToProtoValue[feature.Name] + // arrowArray := types.ProtoValuesToArrowArray(protoValues) + + // fields = append(fields, &arrow.Field{ + // Name: feature.Name, + // Type: arrowArray.DataType(), + // }) + // columns = append(columns, arrowArray) + // } + + // table := array.NewTable( + // arrow.NewSchema(fields, nil), + // columns + // ) + + // pqarrow.WriteTable(table) +} diff --git a/go/cmd/server/server.go b/go/cmd/server/server.go index acb5ff1b62f..5cb7b0e879c 100644 --- a/go/cmd/server/server.go +++ b/go/cmd/server/server.go @@ -2,6 +2,7 @@ package main import ( "context" + "log" "github.com/feast-dev/feast/go/cmd/server/logging" "github.com/feast-dev/feast/go/internal/feast" @@ -32,6 +33,20 @@ func (s *servingServiceServer) GetFeastServingInfo(ctx context.Context, request // Results contains values including the value of the feature, the event timestamp, and feature status in a columnar format. func (s *servingServiceServer) GetOnlineFeatures(ctx context.Context, request *serving.GetOnlineFeaturesRequest) (*serving.GetOnlineFeaturesResponse, error) { featuresOrService, err := s.fs.ParseFeatures(request.GetKind()) + log.Println("Swag money 123") + if featuresOrService.FeatureService != nil { + log.Println("INDSFS") + + log.Println(featuresOrService.FeatureService.Projections[0].Features) + log.Println(featuresOrService.FeatureService.Projections[0].JoinKeyMap) + } + // if featuresOrService.FeatureService != nil { + // if schema, ok := schemaCache[featuresOrService.FeatureService]; !ok { + // generateLogSchemaFromFeatureService(featuresOrService.FeatureService) + // } + + // } + if err != nil { return nil, err } diff --git a/go/internal/feast/basefeatureview.go b/go/internal/feast/basefeatureview.go new file mode 100644 index 00000000000..7a028147b53 --- /dev/null +++ b/go/internal/feast/basefeatureview.go @@ -0,0 +1,43 @@ +package feast + +import ( + "fmt" + + "github.com/feast-dev/feast/go/protos/feast/core" +) + +type BaseFeatureView struct { + name string + features []*Feature + projection *FeatureViewProjection +} + +func NewBaseFeatureView(name string, featureProtos []*core.FeatureSpecV2) *BaseFeatureView { + base := &BaseFeatureView{name: name} + features := make([]*Feature, len(featureProtos)) + for index, featureSpecV2 := range featureProtos { + features[index] = NewFeatureFromProto(featureSpecV2) + } + base.features = features + base.projection = NewFeatureViewProjectionFromDefinition(base) + return base +} + +func (fv *BaseFeatureView) withProjection(projection *FeatureViewProjection) (*BaseFeatureView, error) { + if projection.name != fv.name { + return nil, fmt.Errorf("the projection for the %s FeatureView cannot be applied because it differs "+ + "in name; the projection is named %s and the name indicates which "+ + "FeatureView the projection is for", fv.name, projection.name) + } + features := make(map[string]bool) + for _, feature := range fv.features { + features[feature.name] = true + } + for _, feature := range projection.Features { + if _, ok := features[feature.name]; !ok { + return nil, fmt.Errorf("the projection for %s cannot be applied because it contains %s which the "+ + "FeatureView doesn't have", projection.name, feature.name) + } + } + return &BaseFeatureView{name: fv.name, features: fv.features, projection: projection}, nil +} diff --git a/go/internal/feast/featurestore.go b/go/internal/feast/featurestore.go index e05f2c61e16..4154b8c1e5a 100644 --- a/go/internal/feast/featurestore.go +++ b/go/internal/feast/featurestore.go @@ -34,7 +34,6 @@ type Features struct { // repo config (contents of feature_store.yaml converted to JSON map). func NewFeatureStore(config *registry.RepoConfig, callback transformation.TransformationCallback) (*FeatureStore, error) { onlineStore, err := onlinestore.NewOnlineStore(config) - // offlineStore, err := NewOfflineStore(config) if err != nil { return nil, err } @@ -137,7 +136,6 @@ func (fs *FeatureStore) GetOnlineFeatures( for _, groupRef := range groupedRefs { featureData, err := fs.readFromOnlineStore(ctx, groupRef.EntityKeys, groupRef.FeatureViewNames, groupRef.FeatureNames) - if err != nil { return nil, err } @@ -186,10 +184,6 @@ func (fs *FeatureStore) DestructOnlineStore() { fs.onlineStore.Destruct() } -func (fs *FeatureStore) GetRepoConfig() *RepoConfig { - return fs.config -} - // ParseFeatures parses the kind field of a GetOnlineFeaturesRequest protobuf message // and populates a Features struct with the result. func (fs *FeatureStore) ParseFeatures(kind interface{}) (*Features, error) { diff --git a/go/internal/feast/featureviewprojection.go b/go/internal/feast/featureviewprojection.go new file mode 100644 index 00000000000..89832acdb0b --- /dev/null +++ b/go/internal/feast/featureviewprojection.go @@ -0,0 +1,41 @@ +package feast + +import ( + "github.com/feast-dev/feast/go/protos/feast/core" +) + +type FeatureViewProjection struct { + name string + nameAlias string + Features []*Feature + JoinKeyMap map[string]string +} + +func (fv *FeatureViewProjection) nameToUse() string { + if len(fv.nameAlias) == 0 { + return fv.name + } + return fv.nameAlias +} + +func NewFeatureViewProjectionFromProto(proto *core.FeatureViewProjection) *FeatureViewProjection { + featureProjection := &FeatureViewProjection{name: proto.FeatureViewName, + nameAlias: proto.FeatureViewNameAlias, + JoinKeyMap: proto.JoinKeyMap, + } + + features := make([]*Feature, len(proto.FeatureColumns)) + for index, featureSpecV2 := range proto.FeatureColumns { + features[index] = NewFeatureFromProto(featureSpecV2) + } + featureProjection.Features = features + return featureProjection +} + +func NewFeatureViewProjectionFromDefinition(base *BaseFeatureView) *FeatureViewProjection { + return &FeatureViewProjection{name: base.name, + nameAlias: "", + Features: base.features, + JoinKeyMap: make(map[string]string), + } +} diff --git a/sdk/python/tests/integration/online_store/test_universal_online.py b/sdk/python/tests/integration/online_store/test_universal_online.py index 6762f01a1f3..5e5bd88dbb6 100644 --- a/sdk/python/tests/integration/online_store/test_universal_online.py +++ b/sdk/python/tests/integration/online_store/test_universal_online.py @@ -1094,3 +1094,4 @@ def assert_feature_service_entity_mapping_correctness( entity_rows=entity_rows, full_feature_names=full_feature_names, ) + assert(False) From 79c700c3dd244961b986696e2b3ae0fc2a98a62b Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Mon, 4 Apr 2022 12:51:57 -0700 Subject: [PATCH 32/97] b state Signed-off-by: Kevin Zhang --- go/cmd/server/logging/logging.go | 88 ++++++++++++++----- go/cmd/server/server.go | 1 - go/embedded/online_features.go | 4 +- .../online_store/test_universal_online.py | 8 +- 4 files changed, 75 insertions(+), 26 deletions(-) diff --git a/go/cmd/server/logging/logging.go b/go/cmd/server/logging/logging.go index 85c1c46ccea..4e1595e0385 100644 --- a/go/cmd/server/logging/logging.go +++ b/go/cmd/server/logging/logging.go @@ -6,10 +6,13 @@ import ( "log" "time" + "github.com/apache/arrow/go/arrow" "github.com/apache/arrow/go/arrow/array" + "github.com/apache/arrow/go/arrow/memory" "github.com/feast-dev/feast/go/internal/feast" "github.com/feast-dev/feast/go/protos/feast/serving" "github.com/feast-dev/feast/go/protos/feast/types" + gotypes "github.com/feast-dev/feast/go/types" "google.golang.org/protobuf/types/known/timestamppb" ) @@ -118,29 +121,72 @@ func (s *LoggingService) flushLogsToOfflineStorage(t time.Time) error { return nil } -func (s *LoggingService) getLogInArrowTable(memoryBuffer *MemoryBuffer) (*array.Table, error) { - // input memoryBuffer -> featureColumns - // map[string]*type.Value - - // fields := make([]*arrow.Field, 0) - // columns := make([]array.Interface, 0) - // for idx, feature := range featureService.features { - // feature.Name +func (s *LoggingService) getLogInArrowTable() (*array.Table, error) { + // Memory Buffer is a + if s.memoryBuffer.featureService == nil { + return nil, errors.New("no Feature Service in logging service instantiated") + } + _, requestedFeatureViews, _, _, err := + s.fs.GetFeatureViewsToUseByService(s.memoryBuffer.featureService, false) + if err != nil { + return nil, err + } + entityNameToJoinKeyMap, expectedJoinKeysSet, err := s.fs.GetEntityMaps(requestedFeatureViews) + + columnNameToProtoValueArray := make(map[string][]*types.Value) + columnNameToStatus := make(map[string][]bool) + columnNameToTimestamp := make(map[string][]int64) + entityNameToEntityValues := make(map[string]*types.Value) + for _, fv := range requestedFeatureViews { + // for each feature view we have the features and the entities + // Get the entities from the feature view + // Grab the corresponding join keys and then process using joinkey map to get the entity key related to the features + // For each entity key create a new column + // populate entitynametoentityvalues map + + for idx, featureRef := range fv.FeatureRefs() { + + featureName := featureRef + + // populate the proto value arrays with values from memory buffer in separate columns one for each feature name + if _, ok := columnNameToProtoValueArray[featureName]; !ok { + columnNameToProtoValueArray[featureName] = make([]*types.Value, 0) + columnNameToStatus[featureName] = make([]bool, 0) + columnNameToTimestamp[featureName] = make([]int64, 0) + } + for _, log := range s.memoryBuffer.logs { + columnNameToProtoValueArray[featureName] = append(columnNameToProtoValueArray[featureName], log.FeatureValues[idx]) + if log.FeatureStatuses[idx] == serving.FieldStatus_PRESENT { + + columnNameToStatus[featureName] = append(columnNameToStatus[featureName], true) + } else { + columnNameToStatus[featureName] = append(columnNameToStatus[featureName], false) + } + columnNameToTimestamp[featureName] = append(columnNameToTimestamp[featureName], log.EventTimestamps[idx].AsTime().UnixNano()/int64(time.Millisecond)) + + } + } + } + arrowMemory := memory.NewGoAllocator() - // []*proto.Value = columnNameToProtoValue[feature.Name] - // arrowArray := types.ProtoValuesToArrowArray(protoValues) + fields := make([]*arrow.Field, 0) + columns := make([]array.Interface, 0) + for _, featureView := range s.memoryBuffer.featureService.Projections { + for _, feature := range featureView.Features { - // fields = append(fields, &arrow.Field{ - // Name: feature.Name, - // Type: arrowArray.DataType(), - // }) - // columns = append(columns, arrowArray) - // } + arr := columnNameToProtoValueArray[feature.Name] - // table := array.NewTable( - // arrow.NewSchema(fields, nil), - // columns - // ) + arrowArray, err := gotypes.ProtoValuesToArrowArray(arr, arrowMemory, len(columnNameToProtoValueArray)) + if err != nil { + return nil, err + } - // pqarrow.WriteTable(table) + fields = append(fields, &arrow.Field{ + Name: feature.Name, + Type: arrowArray.DataType(), + }) + columns = append(columns, arrowArray) + } + } + return nil, nil } diff --git a/go/cmd/server/server.go b/go/cmd/server/server.go index 5cb7b0e879c..6f8cb42b29a 100644 --- a/go/cmd/server/server.go +++ b/go/cmd/server/server.go @@ -33,7 +33,6 @@ func (s *servingServiceServer) GetFeastServingInfo(ctx context.Context, request // Results contains values including the value of the feature, the event timestamp, and feature status in a columnar format. func (s *servingServiceServer) GetOnlineFeatures(ctx context.Context, request *serving.GetOnlineFeaturesRequest) (*serving.GetOnlineFeaturesResponse, error) { featuresOrService, err := s.fs.ParseFeatures(request.GetKind()) - log.Println("Swag money 123") if featuresOrService.FeatureService != nil { log.Println("INDSFS") diff --git a/go/embedded/online_features.go b/go/embedded/online_features.go index 217e53d0607..26eab4f48d0 100644 --- a/go/embedded/online_features.go +++ b/go/embedded/online_features.go @@ -3,6 +3,8 @@ package embedded import ( "context" "fmt" + "log" + "github.com/apache/arrow/go/v8/arrow" "github.com/apache/arrow/go/v8/arrow/array" "github.com/apache/arrow/go/v8/arrow/cdata" @@ -14,7 +16,6 @@ import ( "github.com/feast-dev/feast/go/internal/feast/transformation" prototypes "github.com/feast-dev/feast/go/protos/feast/types" "github.com/feast-dev/feast/go/types" - "log" ) type OnlineFeatureService struct { @@ -141,7 +142,6 @@ func (s *OnlineFeatureService) GetOnlineFeatures( if featureServiceName != "" { featureService, err = s.fs.GetFeatureService(featureServiceName) } - resp, err := s.fs.GetOnlineFeatures( context.Background(), featureRefs, diff --git a/sdk/python/tests/integration/online_store/test_universal_online.py b/sdk/python/tests/integration/online_store/test_universal_online.py index 5e5bd88dbb6..f6906208a7b 100644 --- a/sdk/python/tests/integration/online_store/test_universal_online.py +++ b/sdk/python/tests/integration/online_store/test_universal_online.py @@ -703,7 +703,8 @@ def test_online_retrieval_with_go_server( fs = go_environment.feature_store entities, datasets, data_sources = go_data_sources feature_views = construct_universal_feature_views(data_sources, with_odfv=False) - + print("ASdfasdf") + print(feature_views.location) feature_service_entity_mapping = FeatureService( name="entity_mapping", features=[ @@ -1054,12 +1055,14 @@ def assert_feature_service_entity_mapping_correctness( destinations_df, ): if full_feature_names: + print("asdfasdf") feature_service_online_features_dict = get_online_features_dict( environment=environment, features=feature_service, entity_rows=entity_rows, full_feature_names=full_feature_names, ) + print(feature_service_online_features_dict) feature_service_keys = feature_service_online_features_dict.keys() expected_features = [ @@ -1085,6 +1088,8 @@ def assert_feature_service_entity_mapping_correctness( feature_service_online_features_dict[feature_name][i] == df_features[feature_name] ) + + assert(False) else: # using 2 of the same FeatureView without full_feature_names=True will result in collision with pytest.raises(FeatureNameCollisionError): @@ -1094,4 +1099,3 @@ def assert_feature_service_entity_mapping_correctness( entity_rows=entity_rows, full_feature_names=full_feature_names, ) - assert(False) From 6cbf2c2ff29d8694226fc98b47fcf60b8d0b9de3 Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Wed, 6 Apr 2022 15:44:02 -0700 Subject: [PATCH 33/97] Update types to be public Signed-off-by: Kevin Zhang --- go/internal/feast/featureviewprojection.go | 26 ++++++++++++++-------- go/internal/feast/model/feature.go | 6 +++++ go/internal/feast/model/featureservice.go | 10 +++++++++ 3 files changed, 33 insertions(+), 9 deletions(-) diff --git a/go/internal/feast/featureviewprojection.go b/go/internal/feast/featureviewprojection.go index 89832acdb0b..79b5502a8c6 100644 --- a/go/internal/feast/featureviewprojection.go +++ b/go/internal/feast/featureviewprojection.go @@ -5,22 +5,22 @@ import ( ) type FeatureViewProjection struct { - name string - nameAlias string + Name string + NameAlias string Features []*Feature JoinKeyMap map[string]string } func (fv *FeatureViewProjection) nameToUse() string { - if len(fv.nameAlias) == 0 { - return fv.name + if len(fv.NameAlias) == 0 { + return fv.Name } - return fv.nameAlias + return fv.NameAlias } func NewFeatureViewProjectionFromProto(proto *core.FeatureViewProjection) *FeatureViewProjection { - featureProjection := &FeatureViewProjection{name: proto.FeatureViewName, - nameAlias: proto.FeatureViewNameAlias, + featureProjection := &FeatureViewProjection{Name: proto.FeatureViewName, + NameAlias: proto.FeatureViewNameAlias, JoinKeyMap: proto.JoinKeyMap, } @@ -33,9 +33,17 @@ func NewFeatureViewProjectionFromProto(proto *core.FeatureViewProjection) *Featu } func NewFeatureViewProjectionFromDefinition(base *BaseFeatureView) *FeatureViewProjection { - return &FeatureViewProjection{name: base.name, - nameAlias: "", + return &FeatureViewProjection{Name: base.name, + NameAlias: "", Features: base.features, JoinKeyMap: make(map[string]string), } } + +func NewFeatureViewProjection(name string, nameAlias string, features []*Feature, joinKeyMap map[string]string) *FeatureViewProjection { + return &FeatureViewProjection{Name: name, + NameAlias: nameAlias, + Features: features, + JoinKeyMap: joinKeyMap, + } +} diff --git a/go/internal/feast/model/feature.go b/go/internal/feast/model/feature.go index d833a8901b5..b5de880fc7f 100644 --- a/go/internal/feast/model/feature.go +++ b/go/internal/feast/model/feature.go @@ -15,3 +15,9 @@ func NewFeatureFromProto(proto *core.FeatureSpecV2) *Feature { Dtype: proto.ValueType, } } + +func NewFeature(name string, dtype types.ValueType_Enum) *Feature { + return &Feature{Name: name, + Dtype: dtype, + } +} diff --git a/go/internal/feast/model/featureservice.go b/go/internal/feast/model/featureservice.go index 5619dd90426..4d73a3c992a 100644 --- a/go/internal/feast/model/featureservice.go +++ b/go/internal/feast/model/featureservice.go @@ -25,3 +25,13 @@ func NewFeatureServiceFromProto(proto *core.FeatureService) *FeatureService { Projections: projections, } } + +func NewFeatureService(name string, project string, createdTimestamp *timestamppb.Timestamp, lastUpdatedTimestamp *timestamppb.Timestamp, projections []*FeatureViewProjection) *FeatureService { + return &FeatureService{ + name: name, + project: project, + createdTimestamp: createdTimestamp, + lastUpdatedTimestamp: lastUpdatedTimestamp, + Projections: projections, + } +} From aefda92b41a849a1a0b29d1791c10098639fbb05 Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Wed, 6 Apr 2022 15:55:16 -0700 Subject: [PATCH 34/97] Update structs to make fields public Signed-off-by: Kevin Zhang --- go/internal/feast/basefeatureview.go | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/go/internal/feast/basefeatureview.go b/go/internal/feast/basefeatureview.go index 7a028147b53..df9ce762378 100644 --- a/go/internal/feast/basefeatureview.go +++ b/go/internal/feast/basefeatureview.go @@ -7,37 +7,37 @@ import ( ) type BaseFeatureView struct { - name string - features []*Feature - projection *FeatureViewProjection + Name string + Features []*Feature + Projection *FeatureViewProjection } func NewBaseFeatureView(name string, featureProtos []*core.FeatureSpecV2) *BaseFeatureView { - base := &BaseFeatureView{name: name} + base := &BaseFeatureView{Name: name} features := make([]*Feature, len(featureProtos)) for index, featureSpecV2 := range featureProtos { features[index] = NewFeatureFromProto(featureSpecV2) } - base.features = features - base.projection = NewFeatureViewProjectionFromDefinition(base) + base.Features = features + base.Projection = NewFeatureViewProjectionFromDefinition(base) return base } func (fv *BaseFeatureView) withProjection(projection *FeatureViewProjection) (*BaseFeatureView, error) { - if projection.name != fv.name { + if projection.Name != fv.Name { return nil, fmt.Errorf("the projection for the %s FeatureView cannot be applied because it differs "+ "in name; the projection is named %s and the name indicates which "+ - "FeatureView the projection is for", fv.name, projection.name) + "FeatureView the projection is for", fv.Name, projection.Name) } features := make(map[string]bool) - for _, feature := range fv.features { - features[feature.name] = true + for _, feature := range fv.Features { + features[feature.Name] = true } for _, feature := range projection.Features { - if _, ok := features[feature.name]; !ok { + if _, ok := features[feature.Name]; !ok { return nil, fmt.Errorf("the projection for %s cannot be applied because it contains %s which the "+ - "FeatureView doesn't have", projection.name, feature.name) + "FeatureView doesn't have", projection.Name, feature.Name) } } - return &BaseFeatureView{name: fv.name, features: fv.features, projection: projection}, nil + return &BaseFeatureView{Name: fv.Name, Features: fv.Features, Projection: projection}, nil } From 29f72b0453d57ef94b9f04ed38e544824eaec8ab Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Wed, 6 Apr 2022 15:57:59 -0700 Subject: [PATCH 35/97] Fix Signed-off-by: Kevin Zhang --- .../feast/model/ondemandfeatureview.go | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/go/internal/feast/model/ondemandfeatureview.go b/go/internal/feast/model/ondemandfeatureview.go index b8d28c6d7e7..77f1186cc3c 100644 --- a/go/internal/feast/model/ondemandfeatureview.go +++ b/go/internal/feast/model/ondemandfeatureview.go @@ -7,20 +7,34 @@ import ( type OnDemandFeatureView struct { Base *BaseFeatureView +<<<<<<< HEAD:go/internal/feast/model/ondemandfeatureview.go SourceFeatureViewProjections map[string]*FeatureViewProjection SourceRequestDataSources map[string]*core.DataSource_RequestDataOptions +======= + sourceFeatureViewProjections map[string]*FeatureViewProjection + sourceRequestDataSources map[string]*core.DataSource_RequestDataOptions +>>>>>>> 51b50ae5 (Fix):go/internal/feast/ondemandfeatureview.go } func NewOnDemandFeatureViewFromProto(proto *core.OnDemandFeatureView) *OnDemandFeatureView { onDemandFeatureView := &OnDemandFeatureView{Base: NewBaseFeatureView(proto.Spec.Name, proto.Spec.Features), +<<<<<<< HEAD:go/internal/feast/model/ondemandfeatureview.go SourceFeatureViewProjections: make(map[string]*FeatureViewProjection), SourceRequestDataSources: make(map[string]*core.DataSource_RequestDataOptions), +======= + sourceFeatureViewProjections: make(map[string]*FeatureViewProjection), + sourceRequestDataSources: make(map[string]*core.DataSource_RequestDataOptions), +>>>>>>> 51b50ae5 (Fix):go/internal/feast/ondemandfeatureview.go } for sourceName, onDemandSource := range proto.Spec.Sources { if onDemandSourceFeatureView, ok := onDemandSource.Source.(*core.OnDemandSource_FeatureView); ok { featureViewProto := onDemandSourceFeatureView.FeatureView featureView := NewFeatureViewFromProto(featureViewProto) +<<<<<<< HEAD:go/internal/feast/model/ondemandfeatureview.go onDemandFeatureView.SourceFeatureViewProjections[sourceName] = featureView.Base.Projection +======= + onDemandFeatureView.sourceFeatureViewProjections[sourceName] = featureView.Base.Projection +>>>>>>> 51b50ae5 (Fix):go/internal/feast/ondemandfeatureview.go } else if onDemandSourceFeatureViewProjection, ok := onDemandSource.Source.(*core.OnDemandSource_FeatureViewProjection); ok { featureProjectionProto := onDemandSourceFeatureViewProjection.FeatureViewProjection onDemandFeatureView.SourceFeatureViewProjections[sourceName] = NewFeatureViewProjectionFromProto(featureProjectionProto) @@ -35,6 +49,7 @@ func NewOnDemandFeatureViewFromProto(proto *core.OnDemandFeatureView) *OnDemandF return onDemandFeatureView } +<<<<<<< HEAD:go/internal/feast/model/ondemandfeatureview.go func (fs *OnDemandFeatureView) NewWithProjection(projection *FeatureViewProjection) (*OnDemandFeatureView, error) { projectedBase, err := fs.Base.WithProjection(projection) if err != nil { @@ -46,6 +61,12 @@ func (fs *OnDemandFeatureView) NewWithProjection(projection *FeatureViewProjecti SourceRequestDataSources: fs.SourceRequestDataSources, } return featureView, nil +======= +func (fs *OnDemandFeatureView) NewOnDemandFeatureViewFromBase(base *BaseFeatureView) *OnDemandFeatureView { + + featureView := &OnDemandFeatureView{Base: base} + return featureView +>>>>>>> 51b50ae5 (Fix):go/internal/feast/ondemandfeatureview.go } func (fs *OnDemandFeatureView) ProjectWithFeatures(featureNames []string) (*OnDemandFeatureView, error) { From c4dbb84336915e22b1ef14224ca205c4b8b8727a Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Wed, 6 Apr 2022 16:09:44 -0700 Subject: [PATCH 36/97] clean up Signed-off-by: Kevin Zhang --- .../feast/model/ondemandfeatureview.go | 19 ++----------------- 1 file changed, 2 insertions(+), 17 deletions(-) diff --git a/go/internal/feast/model/ondemandfeatureview.go b/go/internal/feast/model/ondemandfeatureview.go index 77f1186cc3c..08db8b416b9 100644 --- a/go/internal/feast/model/ondemandfeatureview.go +++ b/go/internal/feast/model/ondemandfeatureview.go @@ -7,34 +7,20 @@ import ( type OnDemandFeatureView struct { Base *BaseFeatureView -<<<<<<< HEAD:go/internal/feast/model/ondemandfeatureview.go SourceFeatureViewProjections map[string]*FeatureViewProjection SourceRequestDataSources map[string]*core.DataSource_RequestDataOptions -======= - sourceFeatureViewProjections map[string]*FeatureViewProjection - sourceRequestDataSources map[string]*core.DataSource_RequestDataOptions ->>>>>>> 51b50ae5 (Fix):go/internal/feast/ondemandfeatureview.go } func NewOnDemandFeatureViewFromProto(proto *core.OnDemandFeatureView) *OnDemandFeatureView { onDemandFeatureView := &OnDemandFeatureView{Base: NewBaseFeatureView(proto.Spec.Name, proto.Spec.Features), -<<<<<<< HEAD:go/internal/feast/model/ondemandfeatureview.go SourceFeatureViewProjections: make(map[string]*FeatureViewProjection), SourceRequestDataSources: make(map[string]*core.DataSource_RequestDataOptions), -======= - sourceFeatureViewProjections: make(map[string]*FeatureViewProjection), - sourceRequestDataSources: make(map[string]*core.DataSource_RequestDataOptions), ->>>>>>> 51b50ae5 (Fix):go/internal/feast/ondemandfeatureview.go } for sourceName, onDemandSource := range proto.Spec.Sources { if onDemandSourceFeatureView, ok := onDemandSource.Source.(*core.OnDemandSource_FeatureView); ok { featureViewProto := onDemandSourceFeatureView.FeatureView featureView := NewFeatureViewFromProto(featureViewProto) -<<<<<<< HEAD:go/internal/feast/model/ondemandfeatureview.go onDemandFeatureView.SourceFeatureViewProjections[sourceName] = featureView.Base.Projection -======= - onDemandFeatureView.sourceFeatureViewProjections[sourceName] = featureView.Base.Projection ->>>>>>> 51b50ae5 (Fix):go/internal/feast/ondemandfeatureview.go } else if onDemandSourceFeatureViewProjection, ok := onDemandSource.Source.(*core.OnDemandSource_FeatureViewProjection); ok { featureProjectionProto := onDemandSourceFeatureViewProjection.FeatureViewProjection onDemandFeatureView.SourceFeatureViewProjections[sourceName] = NewFeatureViewProjectionFromProto(featureProjectionProto) @@ -49,7 +35,6 @@ func NewOnDemandFeatureViewFromProto(proto *core.OnDemandFeatureView) *OnDemandF return onDemandFeatureView } -<<<<<<< HEAD:go/internal/feast/model/ondemandfeatureview.go func (fs *OnDemandFeatureView) NewWithProjection(projection *FeatureViewProjection) (*OnDemandFeatureView, error) { projectedBase, err := fs.Base.WithProjection(projection) if err != nil { @@ -61,12 +46,12 @@ func (fs *OnDemandFeatureView) NewWithProjection(projection *FeatureViewProjecti SourceRequestDataSources: fs.SourceRequestDataSources, } return featureView, nil -======= +} + func (fs *OnDemandFeatureView) NewOnDemandFeatureViewFromBase(base *BaseFeatureView) *OnDemandFeatureView { featureView := &OnDemandFeatureView{Base: base} return featureView ->>>>>>> 51b50ae5 (Fix):go/internal/feast/ondemandfeatureview.go } func (fs *OnDemandFeatureView) ProjectWithFeatures(featureNames []string) (*OnDemandFeatureView, error) { From 5d7978f04e81c6c8a721b2b84114455d03a2b927 Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Thu, 7 Apr 2022 10:12:45 -0700 Subject: [PATCH 37/97] Fix Signed-off-by: Kevin Zhang --- go/cmd/server/logging/filelogstorage.go | 111 ++++++----- go/cmd/server/logging/filelogstorage_test.go | 145 ++++++-------- go/cmd/server/logging/logging.go | 199 +++++++++++++++---- go/cmd/server/logging/logging_test.go | 181 +++++++++++++++++ go/cmd/server/logging/offlinelogstorage.go | 3 +- go/cmd/server/server.go | 5 +- go/internal/feast/basefeatureview.go | 8 + go/internal/feast/featureviewprojection.go | 4 +- go/internal/feast/model/entity.go | 8 + go/internal/feast/model/featureview.go | 8 + 10 files changed, 495 insertions(+), 177 deletions(-) diff --git a/go/cmd/server/logging/filelogstorage.go b/go/cmd/server/logging/filelogstorage.go index 352ff979f23..72a187ad6c8 100644 --- a/go/cmd/server/logging/filelogstorage.go +++ b/go/cmd/server/logging/filelogstorage.go @@ -2,14 +2,13 @@ package logging import ( "errors" - "fmt" - "log" + "io" "os" "path/filepath" - "time" - "github.com/feast-dev/feast/go/protos/feast/serving" - "github.com/xitongsys/parquet-go/writer" + "github.com/apache/arrow/go/v8/arrow/array" + "github.com/apache/arrow/go/v8/parquet" + "github.com/apache/arrow/go/v8/parquet/pqarrow" ) type FileLogStorage struct { @@ -66,54 +65,70 @@ func CreateOrOpenLogFile(absPath string) (*os.File, error) { } } -func (f *FileLogStorage) FlushToStorage(m *MemoryBuffer) error { - if len(m.logs) == 0 { - return nil - } - var err error +// func (f *FileLogStorage) FlushToStorage(m *MemoryBuffer) error { +// if len(m.logs) == 0 { +// return nil +// } +// var err error +// w, err := CreateOrOpenLogFile(f.path) +// if err != nil { +// return fmt.Errorf("can't create local file with error: %s", err) +// } +// pw, err := writer.NewParquetWriterFromWriter(w, new(ParquetLog), 4) +// if err != nil { +// return fmt.Errorf("can't create parquet writer with error: %s", err) +// } + +// for _, newLog := range m.logs { +// numValues := len(newLog.FeatureValues) +// if numValues != len(newLog.FeatureStatuses) || numValues != len(newLog.EventTimestamps) { +// return errors.New("length of log arrays do not match") +// } +// statuses := make([]bool, numValues) +// timestampsInMillis := make([]int64, numValues) +// featureValues := make([]string, numValues) +// for idx := 0; idx < numValues; idx++ { +// if newLog.FeatureStatuses[idx] == serving.FieldStatus_PRESENT { +// statuses[idx] = true +// } else { +// statuses[idx] = false +// } +// ts := newLog.EventTimestamps[idx] +// timestampsInMillis[idx] = ts.AsTime().UnixNano() / int64(time.Millisecond) +// featureValues[idx] = newLog.FeatureValues[idx].String() +// } +// newParquetLog := ParquetLog{ +// EntityName: newLog.EntityName, +// EntityValue: newLog.EntityValue.String(), +// FeatureNames: newLog.FeatureNames, +// FeatureValues: featureValues, +// FeatureStatuses: statuses, +// EventTimestamps: timestampsInMillis, +// } +// if err = pw.Write(newParquetLog); err != nil { +// return err +// } +// } +// if err = pw.WriteStop(); err != nil { +// return err +// } +// log.Println("Flushed Log to Parquet File Storage") +// w.Close() +// return nil +// } + +func (f *FileLogStorage) FlushToStorage(tbl array.Table) error { w, err := CreateOrOpenLogFile(f.path) + var writer io.Writer = w if err != nil { - return fmt.Errorf("can't create local file with error: %s", err) + return err } - pw, err := writer.NewParquetWriterFromWriter(w, new(ParquetLog), 4) + props := parquet.NewWriterProperties(parquet.WithDictionaryDefault(false)) + arrProps := pqarrow.DefaultWriterProps() + err = pqarrow.WriteTable(tbl, writer, 100, props, arrProps) if err != nil { - return fmt.Errorf("can't create parquet writer with error: %s", err) - } - - for _, newLog := range m.logs { - numValues := len(newLog.FeatureValues) - if numValues != len(newLog.FeatureStatuses) || numValues != len(newLog.EventTimestamps) { - return errors.New("length of log arrays do not match") - } - statuses := make([]bool, numValues) - timestampsInMillis := make([]int64, numValues) - featureValues := make([]string, numValues) - for idx := 0; idx < numValues; idx++ { - if newLog.FeatureStatuses[idx] == serving.FieldStatus_PRESENT { - statuses[idx] = true - } else { - statuses[idx] = false - } - ts := newLog.EventTimestamps[idx] - timestampsInMillis[idx] = ts.AsTime().UnixNano() / int64(time.Millisecond) - featureValues[idx] = newLog.FeatureValues[idx].String() - } - newParquetLog := ParquetLog{ - EntityName: newLog.EntityName, - EntityValue: newLog.EntityValue.String(), - FeatureNames: newLog.FeatureNames, - FeatureValues: featureValues, - FeatureStatuses: statuses, - EventTimestamps: timestampsInMillis, - } - if err = pw.Write(newParquetLog); err != nil { - return err - } - } - if err = pw.WriteStop(); err != nil { return err } - log.Println("Flushed Log to Parquet File Storage") - w.Close() return nil + } diff --git a/go/cmd/server/logging/filelogstorage_test.go b/go/cmd/server/logging/filelogstorage_test.go index 15f51415420..68b4646cdc3 100644 --- a/go/cmd/server/logging/filelogstorage_test.go +++ b/go/cmd/server/logging/filelogstorage_test.go @@ -1,88 +1,73 @@ package logging -import ( - "log" - "os" - "reflect" - "testing" - "time" +// func TestWriteToLogStorage(t *testing.T) { +// offlineStoreConfig := map[string]interface{}{ +// "path": ".", +// } +// fileStore, err := NewFileOfflineStore("test", offlineStoreConfig) +// assert.Nil(t, err) +// ts := timestamppb.New(time.Now()) +// newLog := Log{ +// EntityName: "driver_id", +// EntityValue: &types.Value{Val: &types.Value_Int64Val{Int64Val: 1001}}, +// FeatureNames: []string{"conv_rate", "acc_rate"}, +// FeatureValues: []*types.Value{{Val: &types.Value_FloatVal{FloatVal: 0.2}}, {Val: &types.Value_FloatVal{FloatVal: 0.5}}}, +// FeatureStatuses: []serving.FieldStatus{serving.FieldStatus_PRESENT, serving.FieldStatus_PRESENT}, +// EventTimestamps: []*timestamppb.Timestamp{ts, ts}, +// } +// newLog2 := Log{ +// EntityName: "driver_id", +// EntityValue: &types.Value{Val: &types.Value_Int64Val{Int64Val: 1003}}, +// FeatureNames: []string{"feature4", "feature5"}, +// FeatureValues: []*types.Value{{Val: &types.Value_FloatVal{FloatVal: 0.3}}, {Val: &types.Value_FloatVal{FloatVal: 0.8}}}, +// FeatureStatuses: []serving.FieldStatus{serving.FieldStatus_PRESENT, serving.FieldStatus_PRESENT}, +// EventTimestamps: []*timestamppb.Timestamp{ts, ts}, +// } +// memoryBuffer := MemoryBuffer{ +// logs: []*Log{&newLog, &newLog2}, +// } - "github.com/feast-dev/feast/go/protos/feast/serving" - "github.com/feast-dev/feast/go/protos/feast/types" - "github.com/stretchr/testify/assert" - "github.com/xitongsys/parquet-go-source/local" - "github.com/xitongsys/parquet-go/reader" - "google.golang.org/protobuf/types/known/timestamppb" -) +// err = fileStore.FlushToStorage(&memoryBuffer) +// assert.Nil(t, err) -func TestWriteToLogStorage(t *testing.T) { - offlineStoreConfig := map[string]interface{}{ - "path": ".", - } - fileStore, err := NewFileOfflineStore("test", offlineStoreConfig) - assert.Nil(t, err) - ts := timestamppb.New(time.Now()) - newLog := Log{ - EntityName: "driver_id", - EntityValue: &types.Value{Val: &types.Value_Int64Val{Int64Val: 1001}}, - FeatureNames: []string{"conv_rate", "acc_rate"}, - FeatureValues: []*types.Value{{Val: &types.Value_FloatVal{FloatVal: 0.2}}, {Val: &types.Value_FloatVal{FloatVal: 0.5}}}, - FeatureStatuses: []serving.FieldStatus{serving.FieldStatus_PRESENT, serving.FieldStatus_PRESENT}, - EventTimestamps: []*timestamppb.Timestamp{ts, ts}, - } - newLog2 := Log{ - EntityName: "driver_id", - EntityValue: &types.Value{Val: &types.Value_Int64Val{Int64Val: 1003}}, - FeatureNames: []string{"feature4", "feature5"}, - FeatureValues: []*types.Value{{Val: &types.Value_FloatVal{FloatVal: 0.3}}, {Val: &types.Value_FloatVal{FloatVal: 0.8}}}, - FeatureStatuses: []serving.FieldStatus{serving.FieldStatus_PRESENT, serving.FieldStatus_PRESENT}, - EventTimestamps: []*timestamppb.Timestamp{ts, ts}, - } - memoryBuffer := MemoryBuffer{ - logs: []*Log{&newLog, &newLog2}, - } +// ///read +// fr, err := local.NewLocalFileReader("log.parquet") +// assert.Nil(t, err) - err = fileStore.FlushToStorage(&memoryBuffer) - assert.Nil(t, err) +// pr, err := reader.NewParquetReader(fr, new(ParquetLog), 4) +// if err != nil { +// log.Println("Can't create parquet reader", err) +// return +// } +// num := int(pr.GetNumRows()) +// assert.Equal(t, num, 2) +// logs := make([]ParquetLog, 2) //read 10 rows +// if err = pr.Read(&logs); err != nil { +// log.Println("Read error", err) +// } - ///read - fr, err := local.NewLocalFileReader("log.parquet") - assert.Nil(t, err) +// for i := 0; i < 2; i++ { +// assert.Equal(t, logs[i].EntityName, memoryBuffer.logs[i].EntityName) +// assert.Equal(t, logs[i].EntityValue, memoryBuffer.logs[i].EntityValue.String()) +// assert.True(t, reflect.DeepEqual(logs[i].FeatureNames, memoryBuffer.logs[i].FeatureNames)) +// numValues := len(memoryBuffer.logs[i].FeatureValues) +// assert.Equal(t, numValues, len(logs[i].FeatureValues)) +// assert.Equal(t, numValues, len(logs[i].EventTimestamps)) +// assert.Equal(t, numValues, len(logs[i].FeatureStatuses)) +// for idx := 0; idx < numValues; idx++ { +// assert.Equal(t, logs[i].EventTimestamps[idx], memoryBuffer.logs[i].EventTimestamps[idx].AsTime().UnixNano()/int64(time.Millisecond)) +// if memoryBuffer.logs[i].FeatureStatuses[idx] == serving.FieldStatus_PRESENT { +// assert.True(t, logs[i].FeatureStatuses[idx]) +// } else { +// assert.False(t, logs[i].FeatureStatuses[idx]) +// } +// assert.Equal(t, logs[i].FeatureValues[idx], memoryBuffer.logs[i].FeatureValues[idx].String()) +// } +// } - pr, err := reader.NewParquetReader(fr, new(ParquetLog), 4) - if err != nil { - log.Println("Can't create parquet reader", err) - return - } - num := int(pr.GetNumRows()) - assert.Equal(t, num, 2) - logs := make([]ParquetLog, 2) //read 10 rows - if err = pr.Read(&logs); err != nil { - log.Println("Read error", err) - } +// pr.ReadStop() +// fr.Close() - for i := 0; i < 2; i++ { - assert.Equal(t, logs[i].EntityName, memoryBuffer.logs[i].EntityName) - assert.Equal(t, logs[i].EntityValue, memoryBuffer.logs[i].EntityValue.String()) - assert.True(t, reflect.DeepEqual(logs[i].FeatureNames, memoryBuffer.logs[i].FeatureNames)) - numValues := len(memoryBuffer.logs[i].FeatureValues) - assert.Equal(t, numValues, len(logs[i].FeatureValues)) - assert.Equal(t, numValues, len(logs[i].EventTimestamps)) - assert.Equal(t, numValues, len(logs[i].FeatureStatuses)) - for idx := 0; idx < numValues; idx++ { - assert.Equal(t, logs[i].EventTimestamps[idx], memoryBuffer.logs[i].EventTimestamps[idx].AsTime().UnixNano()/int64(time.Millisecond)) - if memoryBuffer.logs[i].FeatureStatuses[idx] == serving.FieldStatus_PRESENT { - assert.True(t, logs[i].FeatureStatuses[idx]) - } else { - assert.False(t, logs[i].FeatureStatuses[idx]) - } - assert.Equal(t, logs[i].FeatureValues[idx], memoryBuffer.logs[i].FeatureValues[idx].String()) - } - } - - pr.ReadStop() - fr.Close() - - err = os.Remove("log.parquet") - assert.Nil(t, err) -} +// err = os.Remove("log.parquet") +// assert.Nil(t, err) +// } diff --git a/go/cmd/server/logging/logging.go b/go/cmd/server/logging/logging.go index 4e1595e0385..15f3c10526d 100644 --- a/go/cmd/server/logging/logging.go +++ b/go/cmd/server/logging/logging.go @@ -18,9 +18,9 @@ import ( type Log struct { // Example: driver_id, customer_id - EntityName string + EntityName []string // Example: val{int64_val: 5017}, val{int64_val: 1003} - EntityValue *types.Value + EntityValue []*types.Value // Feature names is 1:1 correspondence with featureValue, featureStatus, and timestamp FeatureNames []string @@ -111,8 +111,18 @@ func (s *LoggingService) flushLogsToOfflineStorage(t time.Time) error { } if offlineStoreType == "file" { - s.offlineLogStorage.FlushToStorage(s.memoryBuffer) + //s.offlineLogStorage.FlushToStorage(s.memoryBuffer) //Clean memory buffer + // Convert row level logs array in memory buffer to an array table in columnar + // entities, fvs, odfvs, err := s.GetFcos() + // if err != nil { + // return nil, err + // } + // fcoSchema, err := GetTypesFromFeatureService(s.memoryBuffer.featureService, entities, fvs, odfvs) + // if err != nil { + // return nil, err + // } + // table, err := s.getLogInArrowTable(fcoSchema) s.memoryBuffer.logs = s.memoryBuffer.logs[:0] } else { // Currently don't support any other offline flushing. @@ -121,72 +131,173 @@ func (s *LoggingService) flushLogsToOfflineStorage(t time.Time) error { return nil } -func (s *LoggingService) getLogInArrowTable() (*array.Table, error) { +func (s *LoggingService) getLogInArrowTable(fcoSchema *Schema) (array.Table, error) { // Memory Buffer is a if s.memoryBuffer.featureService == nil { return nil, errors.New("no Feature Service in logging service instantiated") } - _, requestedFeatureViews, _, _, err := - s.fs.GetFeatureViewsToUseByService(s.memoryBuffer.featureService, false) - if err != nil { - return nil, err - } - entityNameToJoinKeyMap, expectedJoinKeysSet, err := s.fs.GetEntityMaps(requestedFeatureViews) + + arrowMemory := memory.NewGoAllocator() columnNameToProtoValueArray := make(map[string][]*types.Value) - columnNameToStatus := make(map[string][]bool) + columnNameToStatus := make(map[string][]int32) columnNameToTimestamp := make(map[string][]int64) - entityNameToEntityValues := make(map[string]*types.Value) - for _, fv := range requestedFeatureViews { - // for each feature view we have the features and the entities + entityNameToEntityValues := make(map[string][]*types.Value) + for _, log := range s.memoryBuffer.logs { + // TODO(kevjumba) get it from the featureview // Get the entities from the feature view // Grab the corresponding join keys and then process using joinkey map to get the entity key related to the features // For each entity key create a new column // populate entitynametoentityvalues map + for entityName, idAndType := range fcoSchema.EntityTypes { + if _, ok := entityNameToEntityValues[entityName]; !ok { + entityNameToEntityValues[entityName] = make([]*types.Value, 0) + } + entityNameToEntityValues[entityName] = append(entityNameToEntityValues[entityName], log.EntityValue[idAndType.index]) + } + for i := 0; i < 0; i++ { + entityKey := log.EntityName[i] + entityVal := log.EntityValue[i] + if _, ok := entityNameToEntityValues[entityKey]; !ok { + entityNameToEntityValues[entityKey] = make([]*types.Value, 0) + } + entityNameToEntityValues[entityKey] = append(columnNameToProtoValueArray[entityKey], entityVal) + } - for idx, featureRef := range fv.FeatureRefs() { - - featureName := featureRef - + for featureName, idAndType := range fcoSchema.FeaturesTypes { // populate the proto value arrays with values from memory buffer in separate columns one for each feature name if _, ok := columnNameToProtoValueArray[featureName]; !ok { columnNameToProtoValueArray[featureName] = make([]*types.Value, 0) - columnNameToStatus[featureName] = make([]bool, 0) + columnNameToStatus[featureName] = make([]int32, 0) columnNameToTimestamp[featureName] = make([]int64, 0) } - for _, log := range s.memoryBuffer.logs { - columnNameToProtoValueArray[featureName] = append(columnNameToProtoValueArray[featureName], log.FeatureValues[idx]) - if log.FeatureStatuses[idx] == serving.FieldStatus_PRESENT { + columnNameToProtoValueArray[featureName] = append(columnNameToProtoValueArray[featureName], log.FeatureValues[idAndType.index]) + columnNameToStatus[featureName] = append(columnNameToStatus[featureName], int32(log.FeatureStatuses[idAndType.index])) + columnNameToTimestamp[featureName] = append(columnNameToTimestamp[featureName], log.EventTimestamps[idAndType.index].AsTime().UnixNano()/int64(time.Millisecond)) + } + } - columnNameToStatus[featureName] = append(columnNameToStatus[featureName], true) - } else { - columnNameToStatus[featureName] = append(columnNameToStatus[featureName], false) - } - columnNameToTimestamp[featureName] = append(columnNameToTimestamp[featureName], log.EventTimestamps[idx].AsTime().UnixNano()/int64(time.Millisecond)) + fields := make([]arrow.Field, 0) + columns := make([]array.Interface, 0) + for name, val := range entityNameToEntityValues { + valArrowArray, err := gotypes.ProtoValuesToArrowArray(val, arrowMemory, len(columnNameToProtoValueArray)) + if err != nil { + return nil, err + } + fields = append(fields, arrow.Field{ + Name: name, + Type: valArrowArray.DataType(), + }) + columns = append(columns, valArrowArray) + } - } + for featureName, _ := range fcoSchema.FeaturesTypes { + + proto_arr := columnNameToProtoValueArray[featureName] + + arrowArray, err := gotypes.ProtoValuesToArrowArray(proto_arr, arrowMemory, len(columnNameToProtoValueArray)) + if err != nil { + return nil, err } + + fields = append(fields, arrow.Field{ + Name: featureName, + Type: arrowArray.DataType(), + }) + columns = append(columns, arrowArray) } - arrowMemory := memory.NewGoAllocator() + log.Println(columns) + schema := arrow.NewSchema( + fields, + nil, + ) + result := array.Record(array.NewRecord(schema, columns, int64(len(s.memoryBuffer.logs)))) + // create an arrow table -> write this to parquet. - fields := make([]*arrow.Field, 0) - columns := make([]array.Interface, 0) - for _, featureView := range s.memoryBuffer.featureService.Projections { - for _, feature := range featureView.Features { + tbl := array.NewTableFromRecords(schema, []array.Record{result}) + // arrow table -> write this to parquet + return array.Table(tbl), nil +} + +type Schema struct { + EntityTypes map[string]*IndexAndType + FeaturesTypes map[string]*IndexAndType + RequestDataTypes map[string]*IndexAndType +} - arr := columnNameToProtoValueArray[feature.Name] +type IndexAndType struct { + dtype types.ValueType_Enum + // index of the fco(entity, feature, request) as it is ordered in logs + index int +} - arrowArray, err := gotypes.ProtoValuesToArrowArray(arr, arrowMemory, len(columnNameToProtoValueArray)) - if err != nil { - return nil, err - } +func GetTypesFromFeatureService(featureService *feast.FeatureService, entities []*feast.Entity, featureViews []*feast.FeatureView, onDemandFeatureViews []*feast.OnDemandFeatureView) (*Schema, error) { + fvs := make(map[string]*feast.FeatureView) + odFvs := make(map[string]*feast.OnDemandFeatureView) - fields = append(fields, &arrow.Field{ - Name: feature.Name, - Type: arrowArray.DataType(), - }) - columns = append(columns, arrowArray) + //featureViews, err := fs.listFeatureViews(hideDummyEntity) + + for _, featureView := range featureViews { + fvs[featureView.Base.Name] = featureView + } + + for _, onDemandFeatureView := range onDemandFeatureViews { + odFvs[onDemandFeatureView.Base.Name] = onDemandFeatureView + } + + entityJoinKeyToType := make(map[string]*IndexAndType) + entityNameToJoinKeyMap := make(map[string]string) + for idx, entity := range entities { + entityNameToJoinKeyMap[entity.Name] = entity.Joinkey + entityJoinKeyToType[entity.Joinkey] = &IndexAndType{ + dtype: entity.Valuetype, + index: idx, + } + } + allFeatureTypes := make(map[string]*IndexAndType) + //allRequestDataTypes := make(map[string]*types.ValueType_Enum) + featureIndex := 0 + for _, featureProjection := range featureService.Projections { + // Create copies of FeatureView that may contains the same *FeatureView but + // each differentiated by a *FeatureViewProjection + featureViewName := featureProjection.Name + if fv, ok := fvs[featureViewName]; ok { + for _, f := range fv.Base.Features { + // add feature to map + allFeatureTypes[f.Name] = &IndexAndType{ + dtype: f.Dtype, + index: featureIndex, + } + featureIndex += 1 + } + } else if _, ok := odFvs[featureViewName]; ok { + // append this -> odfv.getRequestDataSchema() all request data schema + // allRequestDataTypes[featureViewName] = odfv. + featureIndex += 1 + } else { + return nil, fmt.Errorf("no such feature view found in feature service %s", featureViewName) } } - return nil, nil + schema := &Schema{ + EntityTypes: entityJoinKeyToType, + FeaturesTypes: allFeatureTypes, + RequestDataTypes: nil, + } + return schema, nil +} + +func (s *LoggingService) GetFcos() ([]*feast.Entity, []*feast.FeatureView, []*feast.OnDemandFeatureView, error) { + odfvs, err := s.fs.ListOnDemandFeatureViews() + if err != nil { + return nil, nil, nil, err + } + fvs, err := s.fs.ListFeatureViews() + if err != nil { + return nil, nil, nil, err + } + entities, err := s.fs.ListEntities(false) + if err != nil { + return nil, nil, nil, err + } + return entities, fvs, odfvs, nil } diff --git a/go/cmd/server/logging/logging_test.go b/go/cmd/server/logging/logging_test.go index 663d11cf97a..16c409a9195 100644 --- a/go/cmd/server/logging/logging_test.go +++ b/go/cmd/server/logging/logging_test.go @@ -1,15 +1,38 @@ package logging import ( + "log" + "path/filepath" + "reflect" + "runtime" "testing" "time" + "github.com/apache/arrow/go/arrow" + "github.com/apache/arrow/go/arrow/array" + "github.com/feast-dev/feast/go/internal/feast" "github.com/feast-dev/feast/go/protos/feast/serving" + "github.com/feast-dev/feast/go/protos/feast/types" "github.com/stretchr/testify/assert" "google.golang.org/protobuf/types/known/timestamppb" ) +// Return absolute path to the test_repo directory regardless of the working directory +func getRepoPath(basePath string) string { + // Get the file path of this source file, regardless of the working directory + if basePath == "" { + _, filename, _, ok := runtime.Caller(0) + if !ok { + panic("couldn't find file path of the test file") + } + return filepath.Join(filename, "..", "..", "feature_repo") + } else { + return filepath.Join(basePath, "feature_repo") + } +} + func TestLoggingChannelTimeout(t *testing.T) { + // Pregenerated using `feast init`. loggingService, err := NewLoggingService(nil, 1, false) assert.Nil(t, err) assert.Empty(t, loggingService.memoryBuffer.logs) @@ -34,3 +57,161 @@ func TestLoggingChannelTimeout(t *testing.T) { time.Sleep(20 * time.Millisecond) assert.NotNil(t, err) } + +func TestSchemaTypeRetrieval(t *testing.T) { + featureService, entities, featureViews, odfvs := InitializeFeatureRepoVariablesForTest() + schema, err := GetTypesFromFeatureService(featureService, entities, featureViews, odfvs) + assert.Nil(t, err) + log.Println(schema.EntityTypes) + log.Println(schema.FeaturesTypes) + assert.Contains(t, schema.EntityTypes, "driver_id") + assert.True(t, reflect.DeepEqual(schema.EntityTypes["driver_id"], &IndexAndType{ + dtype: types.ValueType_INT64, + index: 0, + })) + + features := []string{"int64", "float32", "int32", "double"} + for idx, featureName := range features { + assert.Contains(t, schema.FeaturesTypes, featureName) + assert.Equal(t, schema.FeaturesTypes[featureName].index, idx) + } + +} + +func TestSerializeToArrowTable(t *testing.T) { + featureService, entities, featureViews, odfvs := InitializeFeatureRepoVariablesForTest() + schema, err := GetTypesFromFeatureService(featureService, entities, featureViews, odfvs) + assert.Nil(t, err) + loggingService, err := NewLoggingService(nil, 1, false) + ts := timestamppb.New(time.Now()) + log.Println("Sdfs") + log1 := Log{ + EntityValue: []*types.Value{ + {Val: &types.Value_Int64Val{Int64Val: 1001}}, + }, + FeatureValues: []*types.Value{ + {Val: &types.Value_Int64Val{Int64Val: 1000}}, + {Val: &types.Value_FloatVal{FloatVal: 0.64}}, + {Val: &types.Value_Int32Val{Int32Val: 55}}, + {Val: &types.Value_DoubleVal{DoubleVal: 0.97}}, + }, + FeatureStatuses: []serving.FieldStatus{ + serving.FieldStatus_PRESENT, + serving.FieldStatus_PRESENT, + serving.FieldStatus_PRESENT, + serving.FieldStatus_PRESENT, + }, + EventTimestamps: []*timestamppb.Timestamp{ + ts, ts, ts, ts, + }, + } + log2 := Log{ + EntityValue: []*types.Value{ + {Val: &types.Value_Int64Val{Int64Val: 1003}}, + }, + FeatureValues: []*types.Value{ + {Val: &types.Value_Int64Val{Int64Val: 1001}}, + {Val: &types.Value_FloatVal{FloatVal: 1.56}}, + {Val: &types.Value_Int32Val{Int32Val: 200}}, + {Val: &types.Value_DoubleVal{DoubleVal: 8.97}}, + }, + FeatureStatuses: []serving.FieldStatus{ + serving.FieldStatus_PRESENT, + serving.FieldStatus_PRESENT, + serving.FieldStatus_PRESENT, + serving.FieldStatus_PRESENT, + }, + EventTimestamps: []*timestamppb.Timestamp{ + ts, ts, ts, ts, + }, + } + memoryBuffer := &MemoryBuffer{ + logs: []*Log{&log1, &log2}, + featureService: featureService, + } + loggingService.memoryBuffer = memoryBuffer + table, err := loggingService.getLogInArrowTable(schema) + defer table.Release() + tr := array.NewTableReader(table, -1) + expected_schema := map[string]arrow.DataType{ + "driver_id": arrow.PrimitiveTypes.Int64, + "int32": arrow.PrimitiveTypes.Int32, + "double": arrow.PrimitiveTypes.Float64, + "int64": arrow.PrimitiveTypes.Int64, + "float32": arrow.PrimitiveTypes.Float32, + } + //mem := memory.NewCheckedAllocator(memory.NewGoAllocator()) + + //expectedReturnedColumns := make([]arrow.Array, 5) + + defer tr.Release() + //returnedColumns := make([][]float64, 5) + for tr.Next() { + rec := tr.Record() + assert.NotNil(t, rec) + log.Println(rec.Schema()) + for _, field := range rec.Schema().Fields() { + assert.Contains(t, expected_schema, field.Name) + assert.Equal(t, field.Type, expected_schema[field.Name]) + } + // log.Println(expected_schema.Fields()) + // assert.True(t, reflect.DeepEqual(rec.Schema().Fields(), expected_schema.Fields())) + // log.Println(rec.Schema()) + // log.Println(rec.NumRows()) + //assert.True(t, reflect.DeepEqual(rec.Columns(), expectedColumns)) + } + + assert.Nil(t, err) +} + +func InitializeFeatureRepoVariablesForTest() (*feast.FeatureService, []*feast.Entity, []*feast.FeatureView, []*feast.OnDemandFeatureView) { + f1 := feast.NewFeature( + "int64", + types.ValueType_INT64, + ) + f2 := feast.NewFeature( + "float32", + types.ValueType_FLOAT, + ) + projection1 := feast.NewFeatureViewProjection( + "featureView1", + "", + []*feast.Feature{f1, f2}, + map[string]string{}, + ) + baseFeatureView1 := feast.CreateBaseFeatureView( + "featureView1", + []*feast.Feature{f1, f2}, + projection1, + ) + featureView1 := feast.CreateFeatureView(baseFeatureView1, nil, map[string]struct{}{}) + entity1 := feast.CreateNewEntity("driver_id", types.ValueType_INT64, "driver_id") + f3 := feast.NewFeature( + "int32", + types.ValueType_INT32, + ) + f4 := feast.NewFeature( + "double", + types.ValueType_DOUBLE, + ) + projection2 := feast.NewFeatureViewProjection( + "featureView2", + "", + []*feast.Feature{f3, f4}, + map[string]string{}, + ) + baseFeatureView2 := feast.CreateBaseFeatureView( + "featureView2", + []*feast.Feature{f3, f4}, + projection2, + ) + featureView2 := feast.CreateFeatureView(baseFeatureView2, nil, map[string]struct{}{}) + featureService := feast.NewFeatureService( + "test_service", + "test_project", + nil, + nil, + []*feast.FeatureViewProjection{projection1, projection2}, + ) + return featureService, []*feast.Entity{entity1}, []*feast.FeatureView{featureView1, featureView2}, []*feast.OnDemandFeatureView{} +} diff --git a/go/cmd/server/logging/offlinelogstorage.go b/go/cmd/server/logging/offlinelogstorage.go index 6e3112b90c2..7f2b36cdaa2 100644 --- a/go/cmd/server/logging/offlinelogstorage.go +++ b/go/cmd/server/logging/offlinelogstorage.go @@ -3,11 +3,12 @@ package logging import ( "errors" + "github.com/apache/arrow/go/v8/arrow/array" "github.com/feast-dev/feast/go/internal/feast" ) type OfflineLogStorage interface { - FlushToStorage(*MemoryBuffer) error + FlushToStorage(array.Table) error } func getOfflineStoreType(offlineStoreConfig map[string]interface{}) (string, bool) { diff --git a/go/cmd/server/server.go b/go/cmd/server/server.go index 6f8cb42b29a..c13c8bafa33 100644 --- a/go/cmd/server/server.go +++ b/go/cmd/server/server.go @@ -77,6 +77,7 @@ func (s *servingServiceServer) GetOnlineFeatures(ctx context.Context, request *s EventTimestamps: make([]*timestamp.Timestamp, 0), } resp.Results = append(resp.Results, vec) + for _, v := range values.Val { entityNames = append(entityNames, name) entityValues = append(entityValues, v) @@ -131,8 +132,8 @@ func generateLogs(s *servingServiceServer, entityNames []string, entityValues [] eventTimestampLogRows[row_idx][idx-1] = results[idx].EventTimestamps[row_idx] } newLog := logging.Log{ - EntityName: entityNames[row_idx], - EntityValue: entityValues[row_idx], + EntityName: entityNames, + EntityValue: entityValues, FeatureNames: featureNames, FeatureValues: featureValueLogRows[row_idx], FeatureStatuses: featureStatusLogRows[row_idx], diff --git a/go/internal/feast/basefeatureview.go b/go/internal/feast/basefeatureview.go index df9ce762378..6f9776bb728 100644 --- a/go/internal/feast/basefeatureview.go +++ b/go/internal/feast/basefeatureview.go @@ -41,3 +41,11 @@ func (fv *BaseFeatureView) withProjection(projection *FeatureViewProjection) (*B } return &BaseFeatureView{Name: fv.Name, Features: fv.Features, Projection: projection}, nil } + +func CreateBaseFeatureView(name string, features []*Feature, projection *FeatureViewProjection) *BaseFeatureView { + return &BaseFeatureView{ + Name: name, + Features: features, + Projection: projection, + } +} diff --git a/go/internal/feast/featureviewprojection.go b/go/internal/feast/featureviewprojection.go index 79b5502a8c6..52f498b2d00 100644 --- a/go/internal/feast/featureviewprojection.go +++ b/go/internal/feast/featureviewprojection.go @@ -33,9 +33,9 @@ func NewFeatureViewProjectionFromProto(proto *core.FeatureViewProjection) *Featu } func NewFeatureViewProjectionFromDefinition(base *BaseFeatureView) *FeatureViewProjection { - return &FeatureViewProjection{Name: base.name, + return &FeatureViewProjection{Name: base.Name, NameAlias: "", - Features: base.features, + Features: base.Features, JoinKeyMap: make(map[string]string), } } diff --git a/go/internal/feast/model/entity.go b/go/internal/feast/model/entity.go index ac3a5d5f26e..d1d817756f9 100644 --- a/go/internal/feast/model/entity.go +++ b/go/internal/feast/model/entity.go @@ -17,3 +17,11 @@ func NewEntityFromProto(proto *core.Entity) *Entity { JoinKey: proto.Spec.JoinKey, } } + +func CreateNewEntity(name string, valueType types.ValueType_Enum, joinKey string) *Entity { + return &Entity{ + Name: name, + Valuetype: valueType, + Joinkey: joinKey, + } +} diff --git a/go/internal/feast/model/featureview.go b/go/internal/feast/model/featureview.go index bdb86f0ace5..616df48e946 100644 --- a/go/internal/feast/model/featureview.go +++ b/go/internal/feast/model/featureview.go @@ -44,3 +44,11 @@ func (fs *FeatureView) NewFeatureViewFromBase(base *BaseFeatureView) *FeatureVie } return featureView } + +func CreateFeatureView(base *BaseFeatureView, ttl *durationpb.Duration, entities map[string]struct{}) *FeatureView { + return &FeatureView{ + Base: base, + ttl: ttl, + Entities: entities, + } +} From 686c58395378ab99463619f1528f606abde7d05e Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Thu, 7 Apr 2022 10:13:00 -0700 Subject: [PATCH 38/97] Fix go Signed-off-by: Kevin Zhang --- go.mod | 22 ++++++++++++++++++---- go.sum | 21 +++++++++++++++++++++ 2 files changed, 39 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index 319605eda01..a6375dd11ee 100644 --- a/go.mod +++ b/go.mod @@ -19,7 +19,12 @@ require ( require ( github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c // indirect github.com/andybalholm/brotli v1.0.4 // indirect +<<<<<<< HEAD github.com/apache/thrift v0.15.0 // indirect +======= + github.com/apache/arrow/go/v8 v8.0.0-20220405223432-9fa34df27eb1 // indirect + github.com/apache/thrift v0.16.0 // indirect +>>>>>>> e0ce4335 (Fix go) github.com/cespare/xxhash/v2 v2.1.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect @@ -27,6 +32,7 @@ require ( github.com/golang/snappy v0.0.4 // indirect github.com/gonuts/commander v0.1.0 // indirect github.com/gonuts/flag v0.1.0 // indirect +<<<<<<< HEAD <<<<<<< HEAD github.com/google/flatbuffers v2.0.5+incompatible // indirect github.com/klauspost/asmfmt v1.3.1 // indirect @@ -45,15 +51,23 @@ require ( ======= >>>>>>> 8a7ccbe5 (semi working state) github.com/google/flatbuffers v2.0.0+incompatible // indirect +======= + github.com/google/flatbuffers v2.0.5+incompatible // indirect +>>>>>>> e0ce4335 (Fix go) github.com/google/go-cmp v0.5.7 // indirect + github.com/klauspost/asmfmt v1.3.1 // indirect github.com/klauspost/compress v1.15.1 // indirect - github.com/pierrec/lz4/v4 v4.1.9 // indirect + github.com/klauspost/cpuid/v2 v2.0.9 // indirect + github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 // indirect + github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 // indirect + github.com/pierrec/lz4/v4 v4.1.12 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - golang.org/x/exp v0.0.0-20211028214138-64b4c8e87d1a // indirect + github.com/zeebo/xxh3 v1.0.1 // indirect + golang.org/x/exp v0.0.0-20211216164055-b2b84827b756 // indirect golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3 // indirect - golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f // indirect - golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359 // indirect + golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd // indirect + golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27 // indirect golang.org/x/text v0.3.7 // indirect golang.org/x/tools v0.1.10 // indirect golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect diff --git a/go.sum b/go.sum index b17f01aa905..3f87cb76067 100644 --- a/go.sum +++ b/go.sum @@ -214,6 +214,8 @@ github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvW github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/asmfmt v1.3.1 h1:7xZi1N7s9gTLbqiM8KUv8TLyysavbTRGBT5/ly0bRtw= github.com/klauspost/asmfmt v1.3.1/go.mod h1:AG8TuvYojzulgDAMCnYn50l/5QV3Bs/tp6j0HLHbNSE= +github.com/klauspost/compress v1.9.7/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= +github.com/klauspost/compress v1.13.1/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= github.com/klauspost/compress v1.14.2/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/compress v1.15.1 h1:y9FcTHGyrebwfP0ZZqFiaxTaiDnUrGkJkI+f583BL1A= github.com/klauspost/compress v1.15.1/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= @@ -294,6 +296,9 @@ github.com/phpdave11/gofpdi v1.0.12/go.mod h1:vBmVV0Do6hSBHC8uKUQ71JGW+ZGQq74llk github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc= github.com/pierrec/lz4 v2.0.5+incompatible h1:2xWsjqPFWcplujydGg4WmhC/6fZqK42wMM8aXeqhl0I= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= +github.com/pierrec/lz4/v4 v4.1.8/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= +github.com/pierrec/lz4/v4 v4.1.9 h1:xkrjwpOP5xg1k4Nn4GX4a4YFGhscyQL/3EddJ1Xxqm8= +github.com/pierrec/lz4/v4 v4.1.9/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pierrec/lz4/v4 v4.1.12 h1:44l88ehTZAUGW4VlO1QC4zkilL99M6Y9MXNwEs0uzP8= github.com/pierrec/lz4/v4 v4.1.12/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -361,6 +366,7 @@ github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijb github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/zeebo/xxh3 v1.0.1 h1:FMSRIbkrLikb/0hZxmltpg84VkqDAT5M8ufXynuhXsI= github.com/zeebo/xxh3 v1.0.1/go.mod h1:8VHV24/3AZLn3b6Mlp/KuC33LWH687Wq6EnziEB+rsA= @@ -394,6 +400,8 @@ golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -402,6 +410,14 @@ golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190731235908-ec7cb31e5a56/go.mod h1:JhuoJpWY28nO4Vef9tZUw9qufEGTyX1+7lmHxV5q5G4= golang.org/x/exp v0.0.0-20191002040644-a1355ae1e2c3/go.mod h1:NOZ3BPKG0ec/BKJQgnvsSFpcKLM5xXVWnvZS97DWHgE= +golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= +golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= +golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/exp v0.0.0-20211028214138-64b4c8e87d1a h1:9kUIHyUjWEuW2MdtaYvvEXTDs2eNylNVAt/ui66QGOg= +golang.org/x/exp v0.0.0-20211028214138-64b4c8e87d1a/go.mod h1:a3o/VtDNHN+dCVLEpzjjUHOzR+Ln3DHX056ZPzoZGGA= golang.org/x/exp v0.0.0-20211216164055-b2b84827b756 h1:/5Bs7sWi0i3rOVO5KnM55OwugpsD4bRW1zywKoZjbkI= golang.org/x/exp v0.0.0-20211216164055-b2b84827b756/go.mod h1:b9TAUYHmRtqA6klRHApnXMnj+OyLce4yF5cZCUbk2ps= golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= @@ -431,6 +447,8 @@ golang.org/x/mod v0.6.0-dev.0.20211013180041-c96bc1413d57 h1:LQmS1nU0twXLA96Kt7U golang.org/x/mod v0.6.0-dev.0.20211013180041-c96bc1413d57/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.5.1-0.20210830214625-1b1db11ec8f4/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= +golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= +golang.org/x/mod v0.6.0-dev.0.20211013180041-c96bc1413d57/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3 h1:kQgndtyPBW/JIYERgdxfwMYh3AVStj88WQTlNDi2a+o= golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -563,6 +581,9 @@ google.golang.org/genproto v0.0.0-20190530194941-fb225487d101/go.mod h1:z3L6/3dT google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20210630183607-d20f26d13c79/go.mod h1:yiaVoXHpRzHGyxV3o4DktVWY4mSUErTKaeEOq6C3t3U= +google.golang.org/genproto v0.0.0-20220118154757-00ab72f36ad5 h1:zzNejm+EgrbLfDZ6lu9Uud2IVvHySPl8vQzf04laR5Q= +google.golang.org/genproto v0.0.0-20220118154757-00ab72f36ad5/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20220126215142-9970aeb2e350 h1:YxHp5zqIcAShDEvRr5/0rVESVS+njYF68PSdazrNLJo= google.golang.org/genproto v0.0.0-20220126215142-9970aeb2e350/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= From 884b014a539f0460522449c10cf928898fffeaaa Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Thu, 7 Apr 2022 15:03:08 -0700 Subject: [PATCH 39/97] Fix issues Signed-off-by: Kevin Zhang --- go/cmd/server/logging/logging_test.go | 48 +++++++++++++++++++++------ go/types/typeconversion.go | 1 + 2 files changed, 39 insertions(+), 10 deletions(-) diff --git a/go/cmd/server/logging/logging_test.go b/go/cmd/server/logging/logging_test.go index 16c409a9195..bc2e80d7eeb 100644 --- a/go/cmd/server/logging/logging_test.go +++ b/go/cmd/server/logging/logging_test.go @@ -8,6 +8,8 @@ import ( "testing" "time" + gotypes "github.com/feast-dev/feast/go/types" + "github.com/apache/arrow/go/arrow" "github.com/apache/arrow/go/arrow/array" "github.com/feast-dev/feast/go/internal/feast" @@ -83,8 +85,8 @@ func TestSerializeToArrowTable(t *testing.T) { schema, err := GetTypesFromFeatureService(featureService, entities, featureViews, odfvs) assert.Nil(t, err) loggingService, err := NewLoggingService(nil, 1, false) + assert.Nil(t, err) ts := timestamppb.New(time.Now()) - log.Println("Sdfs") log1 := Log{ EntityValue: []*types.Value{ {Val: &types.Value_Int64Val{Int64Val: 1001}}, @@ -140,12 +142,26 @@ func TestSerializeToArrowTable(t *testing.T) { "int64": arrow.PrimitiveTypes.Int64, "float32": arrow.PrimitiveTypes.Float32, } - //mem := memory.NewCheckedAllocator(memory.NewGoAllocator()) - //expectedReturnedColumns := make([]arrow.Array, 5) + expected_columns := map[string]*types.RepeatedValue{ + "double": { + Val: []*types.Value{{Val: &types.Value_DoubleVal{DoubleVal: 0.97}}, + {Val: &types.Value_DoubleVal{DoubleVal: 8.97}}}}, + "driver_id": { + Val: []*types.Value{{Val: &types.Value_Int64Val{Int64Val: 1001}}, + {Val: &types.Value_Int64Val{Int64Val: 1003}}}}, + "float32": { + Val: []*types.Value{{Val: &types.Value_FloatVal{FloatVal: 0.64}}, + {Val: &types.Value_FloatVal{FloatVal: 1.56}}}}, + "int32": { + Val: []*types.Value{{Val: &types.Value_Int32Val{Int32Val: 55}}, + {Val: &types.Value_Int32Val{Int32Val: 200}}}}, + "int64": { + Val: []*types.Value{{Val: &types.Value_Int64Val{Int64Val: 1000}}, + {Val: &types.Value_Int64Val{Int64Val: 1001}}}}, + } defer tr.Release() - //returnedColumns := make([][]float64, 5) for tr.Next() { rec := tr.Record() assert.NotNil(t, rec) @@ -154,16 +170,28 @@ func TestSerializeToArrowTable(t *testing.T) { assert.Contains(t, expected_schema, field.Name) assert.Equal(t, field.Type, expected_schema[field.Name]) } - // log.Println(expected_schema.Fields()) - // assert.True(t, reflect.DeepEqual(rec.Schema().Fields(), expected_schema.Fields())) - // log.Println(rec.Schema()) - // log.Println(rec.NumRows()) - //assert.True(t, reflect.DeepEqual(rec.Columns(), expectedColumns)) + values, err := GetProtoFromRecord(rec) + + assert.Nil(t, err) + assert.True(t, reflect.DeepEqual(values, expected_columns)) + log.Println(values) } assert.Nil(t, err) } - +func GetProtoFromRecord(rec array.Record) (map[string]*types.RepeatedValue, error) { + r := make(map[string]*types.RepeatedValue) + schema := rec.Schema() + for idx, column := range rec.Columns() { + field := schema.Field(idx) + values, err := gotypes.ArrowValuesToProtoValues(column) + if err != nil { + return nil, err + } + r[field.Name] = &types.RepeatedValue{Val: values} + } + return r, nil +} func InitializeFeatureRepoVariablesForTest() (*feast.FeatureService, []*feast.Entity, []*feast.FeatureView, []*feast.OnDemandFeatureView) { f1 := feast.NewFeature( "int64", diff --git a/go/types/typeconversion.go b/go/types/typeconversion.go index 798d352b375..fc223b02c26 100644 --- a/go/types/typeconversion.go +++ b/go/types/typeconversion.go @@ -2,6 +2,7 @@ package types import ( "fmt" + "github.com/apache/arrow/go/v8/arrow" "github.com/apache/arrow/go/v8/arrow/array" "github.com/apache/arrow/go/v8/arrow/memory" From c02cf83bbf24e4c9bdc18b19e6d34e6e74166fb9 Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Thu, 7 Apr 2022 15:05:05 -0700 Subject: [PATCH 40/97] Fix Signed-off-by: Kevin Zhang --- go/internal/feast/featurestore_test.go | 176 +++++++++++++++++++++++++ 1 file changed, 176 insertions(+) diff --git a/go/internal/feast/featurestore_test.go b/go/internal/feast/featurestore_test.go index 6b8052c8034..33c1ed4660e 100644 --- a/go/internal/feast/featurestore_test.go +++ b/go/internal/feast/featurestore_test.go @@ -85,5 +85,181 @@ func TestGetOnlineFeaturesRedis(t *testing.T) { response, err := fs.GetOnlineFeatures( ctx, featureNames, nil, entities, map[string]*types.RepeatedValue{}, true) assert.Nil(t, err) +<<<<<<< HEAD assert.Len(t, response, 4) // 3 Features + 1 entity = 4 columns (feature vectors) in response +======= + assert.Len(t, response, 4) // 3 features + 1 entity = 4 columns (feature vectors) in response +} + +func TestGroupingFeatureRefs(t *testing.T) { + viewA := &FeatureView{ + Base: &BaseFeatureView{ + Name: "viewA", + Projection: &FeatureViewProjection{ + NameAlias: "aliasViewA", + }, + }, + Entities: map[string]struct{}{"driver": {}, "customer": {}}, + } + viewB := &FeatureView{ + Base: &BaseFeatureView{Name: "viewB"}, + Entities: map[string]struct{}{"driver": {}, "customer": {}}, + } + viewC := &FeatureView{ + Base: &BaseFeatureView{Name: "viewC"}, + Entities: map[string]struct{}{"driver": {}}, + } + viewD := &FeatureView{ + Base: &BaseFeatureView{Name: "viewD"}, + Entities: map[string]struct{}{"customer": {}}, + } + refGroups, _ := groupFeatureRefs( + []*featureViewAndRefs{ + {view: viewA, featureRefs: []string{"featureA", "featureB"}}, + {view: viewB, featureRefs: []string{"featureC", "featureD"}}, + {view: viewC, featureRefs: []string{"featureE"}}, + {view: viewD, featureRefs: []string{"featureF"}}, + }, + map[string]*types.RepeatedValue{ + "driver_id": {Val: []*types.Value{ + {Val: &types.Value_Int32Val{Int32Val: 0}}, + {Val: &types.Value_Int32Val{Int32Val: 0}}, + {Val: &types.Value_Int32Val{Int32Val: 1}}, + {Val: &types.Value_Int32Val{Int32Val: 1}}, + {Val: &types.Value_Int32Val{Int32Val: 1}}, + }}, + "customer_id": {Val: []*types.Value{ + {Val: &types.Value_Int32Val{Int32Val: 1}}, + {Val: &types.Value_Int32Val{Int32Val: 2}}, + {Val: &types.Value_Int32Val{Int32Val: 3}}, + {Val: &types.Value_Int32Val{Int32Val: 3}}, + {Val: &types.Value_Int32Val{Int32Val: 4}}, + }}, + }, + map[string]string{ + "driver": "driver_id", + "customer": "customer_id", + }, + true, + ) + + assert.Len(t, refGroups, 3) + + // Group 1 + assert.Equal(t, []string{"featureA", "featureB", "featureC", "featureD"}, + refGroups["customer_id,driver_id"].featureNames) + assert.Equal(t, []string{"viewA", "viewA", "viewB", "viewB"}, + refGroups["customer_id,driver_id"].featureViewNames) + assert.Equal(t, []string{ + "aliasViewA__featureA", "aliasViewA__featureB", + "viewB__featureC", "viewB__featureD"}, + refGroups["customer_id,driver_id"].aliasedFeatureNames) + for _, group := range [][]int{{0}, {1}, {2, 3}, {4}} { + assert.Contains(t, refGroups["customer_id,driver_id"].indices, group) + } + + // Group2 + assert.Equal(t, []string{"featureE"}, + refGroups["driver_id"].featureNames) + for _, group := range [][]int{{0, 1}, {2, 3, 4}} { + assert.Contains(t, refGroups["driver_id"].indices, group) + } + + // Group3 + assert.Equal(t, []string{"featureF"}, + refGroups["customer_id"].featureNames) + + for _, group := range [][]int{{0}, {1}, {2, 3}, {4}} { + assert.Contains(t, refGroups["customer_id"].indices, group) + } + +} + +func TestGroupingFeatureRefsWithJoinKeyAliases(t *testing.T) { + viewA := &FeatureView{ + Base: &BaseFeatureView{ + Name: "viewA", + Projection: &FeatureViewProjection{ + Name: "viewA", + JoinKeyMap: map[string]string{"location_id": "destination_id"}, + }, + }, + Entities: map[string]struct{}{"location": {}}, + } + viewB := &FeatureView{ + Base: &BaseFeatureView{Name: "viewB"}, + Entities: map[string]struct{}{"location": {}}, + } + + refGroups, _ := groupFeatureRefs( + []*featureViewAndRefs{ + {view: viewA, featureRefs: []string{"featureA", "featureB"}}, + {view: viewB, featureRefs: []string{"featureC", "featureD"}}, + }, + map[string]*types.RepeatedValue{ + "location_id": {Val: []*types.Value{ + {Val: &types.Value_Int32Val{Int32Val: 0}}, + {Val: &types.Value_Int32Val{Int32Val: 0}}, + {Val: &types.Value_Int32Val{Int32Val: 1}}, + {Val: &types.Value_Int32Val{Int32Val: 1}}, + {Val: &types.Value_Int32Val{Int32Val: 1}}, + }}, + "destination_id": {Val: []*types.Value{ + {Val: &types.Value_Int32Val{Int32Val: 1}}, + {Val: &types.Value_Int32Val{Int32Val: 2}}, + {Val: &types.Value_Int32Val{Int32Val: 3}}, + {Val: &types.Value_Int32Val{Int32Val: 3}}, + {Val: &types.Value_Int32Val{Int32Val: 4}}, + }}, + }, + map[string]string{ + "location": "location_id", + }, + true, + ) + + assert.Len(t, refGroups, 2) + + assert.Equal(t, []string{"featureA", "featureB"}, + refGroups["location_id[destination_id]"].featureNames) + for _, group := range [][]int{{0}, {1}, {2, 3}, {4}} { + assert.Contains(t, refGroups["location_id[destination_id]"].indices, group) + } + + assert.Equal(t, []string{"featureC", "featureD"}, + refGroups["location_id"].featureNames) + for _, group := range [][]int{{0, 1}, {2, 3, 4}} { + assert.Contains(t, refGroups["location_id"].indices, group) + } + +} + +func TestGroupingFeatureRefsWithMissingKey(t *testing.T) { + viewA := &FeatureView{ + Base: &BaseFeatureView{ + Name: "viewA", + Projection: &FeatureViewProjection{ + Name: "viewA", + JoinKeyMap: map[string]string{"location_id": "destination_id"}, + }, + }, + Entities: map[string]struct{}{"location": {}}, + } + + _, err := groupFeatureRefs( + []*featureViewAndRefs{ + {view: viewA, featureRefs: []string{"featureA", "featureB"}}, + }, + map[string]*types.RepeatedValue{ + "location_id": {Val: []*types.Value{ + {Val: &types.Value_Int32Val{Int32Val: 0}}, + }}, + }, + map[string]string{ + "location": "location_id", + }, + true, + ) + assert.Errorf(t, err, "key destination_id is missing in provided entity rows") +>>>>>>> 14784f07 (Fix) } From e89abc1e800336d4320ca24f81cbff45f8c0679e Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Thu, 7 Apr 2022 15:11:18 -0700 Subject: [PATCH 41/97] Fix tests Signed-off-by: Kevin Zhang --- go/cmd/server/logging/logging.go | 13 ------------- go/cmd/server/logging/logging_test.go | 7 ------- 2 files changed, 20 deletions(-) diff --git a/go/cmd/server/logging/logging.go b/go/cmd/server/logging/logging.go index 15f3c10526d..5d5bae36903 100644 --- a/go/cmd/server/logging/logging.go +++ b/go/cmd/server/logging/logging.go @@ -17,14 +17,9 @@ import ( ) type Log struct { - // Example: driver_id, customer_id - EntityName []string // Example: val{int64_val: 5017}, val{int64_val: 1003} EntityValue []*types.Value - // Feature names is 1:1 correspondence with featureValue, featureStatus, and timestamp - FeatureNames []string - FeatureValues []*types.Value FeatureStatuses []serving.FieldStatus EventTimestamps []*timestamppb.Timestamp @@ -155,14 +150,6 @@ func (s *LoggingService) getLogInArrowTable(fcoSchema *Schema) (array.Table, err } entityNameToEntityValues[entityName] = append(entityNameToEntityValues[entityName], log.EntityValue[idAndType.index]) } - for i := 0; i < 0; i++ { - entityKey := log.EntityName[i] - entityVal := log.EntityValue[i] - if _, ok := entityNameToEntityValues[entityKey]; !ok { - entityNameToEntityValues[entityKey] = make([]*types.Value, 0) - } - entityNameToEntityValues[entityKey] = append(columnNameToProtoValueArray[entityKey], entityVal) - } for featureName, idAndType := range fcoSchema.FeaturesTypes { // populate the proto value arrays with values from memory buffer in separate columns one for each feature name diff --git a/go/cmd/server/logging/logging_test.go b/go/cmd/server/logging/logging_test.go index bc2e80d7eeb..f4d3b896d56 100644 --- a/go/cmd/server/logging/logging_test.go +++ b/go/cmd/server/logging/logging_test.go @@ -1,7 +1,6 @@ package logging import ( - "log" "path/filepath" "reflect" "runtime" @@ -40,7 +39,6 @@ func TestLoggingChannelTimeout(t *testing.T) { assert.Empty(t, loggingService.memoryBuffer.logs) ts := timestamppb.New(time.Now()) newLog := Log{ - FeatureNames: []string{"feature1", "feature2"}, FeatureStatuses: []serving.FieldStatus{serving.FieldStatus_PRESENT}, EventTimestamps: []*timestamppb.Timestamp{ts, ts}, } @@ -50,7 +48,6 @@ func TestLoggingChannelTimeout(t *testing.T) { newTs := timestamppb.New(time.Now()) newLog2 := Log{ - FeatureNames: []string{"feature4", "feature5"}, FeatureStatuses: []serving.FieldStatus{serving.FieldStatus_PRESENT}, EventTimestamps: []*timestamppb.Timestamp{newTs, newTs}, } @@ -64,8 +61,6 @@ func TestSchemaTypeRetrieval(t *testing.T) { featureService, entities, featureViews, odfvs := InitializeFeatureRepoVariablesForTest() schema, err := GetTypesFromFeatureService(featureService, entities, featureViews, odfvs) assert.Nil(t, err) - log.Println(schema.EntityTypes) - log.Println(schema.FeaturesTypes) assert.Contains(t, schema.EntityTypes, "driver_id") assert.True(t, reflect.DeepEqual(schema.EntityTypes["driver_id"], &IndexAndType{ dtype: types.ValueType_INT64, @@ -165,7 +160,6 @@ func TestSerializeToArrowTable(t *testing.T) { for tr.Next() { rec := tr.Record() assert.NotNil(t, rec) - log.Println(rec.Schema()) for _, field := range rec.Schema().Fields() { assert.Contains(t, expected_schema, field.Name) assert.Equal(t, field.Type, expected_schema[field.Name]) @@ -174,7 +168,6 @@ func TestSerializeToArrowTable(t *testing.T) { assert.Nil(t, err) assert.True(t, reflect.DeepEqual(values, expected_columns)) - log.Println(values) } assert.Nil(t, err) From 559bb8304ffa1ca25d306cd8872bdb53387626d0 Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Thu, 7 Apr 2022 16:27:42 -0700 Subject: [PATCH 42/97] Fix Signed-off-by: Kevin Zhang --- go/cmd/server/logging/filelogstorage_test.go | 72 ++++++++++++++++++++ go/cmd/server/logging/logging.go | 9 ++- go/cmd/server/logging/logging_test.go | 6 +- 3 files changed, 79 insertions(+), 8 deletions(-) diff --git a/go/cmd/server/logging/filelogstorage_test.go b/go/cmd/server/logging/filelogstorage_test.go index 68b4646cdc3..a9d155a0763 100644 --- a/go/cmd/server/logging/filelogstorage_test.go +++ b/go/cmd/server/logging/filelogstorage_test.go @@ -1,5 +1,16 @@ package logging +import ( + "testing" + "time" + + "github.com/apache/arrow/go/v8/arrow/array" + "github.com/feast-dev/feast/go/protos/feast/serving" + "github.com/feast-dev/feast/go/protos/feast/types" + "github.com/stretchr/testify/assert" + "google.golang.org/protobuf/types/known/timestamppb" +) + // func TestWriteToLogStorage(t *testing.T) { // offlineStoreConfig := map[string]interface{}{ // "path": ".", @@ -71,3 +82,64 @@ package logging // err = os.Remove("log.parquet") // assert.Nil(t, err) // } + +func TestFlushToStorage(t *testing.T) { + featureService, entities, featureViews, odfvs := InitializeFeatureRepoVariablesForTest() + schema, err := GetTypesFromFeatureService(featureService, entities, featureViews, odfvs) + assert.Nil(t, err) + loggingService, err := NewLoggingService(nil, 1, false) + assert.Nil(t, err) + ts := timestamppb.New(time.Now()) + log1 := Log{ + EntityValue: []*types.Value{ + {Val: &types.Value_Int64Val{Int64Val: 1001}}, + }, + FeatureValues: []*types.Value{ + {Val: &types.Value_Int64Val{Int64Val: 1000}}, + {Val: &types.Value_FloatVal{FloatVal: 0.64}}, + {Val: &types.Value_Int32Val{Int32Val: 55}}, + {Val: &types.Value_DoubleVal{DoubleVal: 0.97}}, + }, + FeatureStatuses: []serving.FieldStatus{ + serving.FieldStatus_PRESENT, + serving.FieldStatus_PRESENT, + serving.FieldStatus_PRESENT, + serving.FieldStatus_PRESENT, + }, + EventTimestamps: []*timestamppb.Timestamp{ + ts, ts, ts, ts, + }, + } + log2 := Log{ + EntityValue: []*types.Value{ + {Val: &types.Value_Int64Val{Int64Val: 1003}}, + }, + FeatureValues: []*types.Value{ + {Val: &types.Value_Int64Val{Int64Val: 1001}}, + {Val: &types.Value_FloatVal{FloatVal: 1.56}}, + {Val: &types.Value_Int32Val{Int32Val: 200}}, + {Val: &types.Value_DoubleVal{DoubleVal: 8.97}}, + }, + FeatureStatuses: []serving.FieldStatus{ + serving.FieldStatus_PRESENT, + serving.FieldStatus_PRESENT, + serving.FieldStatus_PRESENT, + serving.FieldStatus_PRESENT, + }, + EventTimestamps: []*timestamppb.Timestamp{ + ts, ts, ts, ts, + }, + } + memoryBuffer := &MemoryBuffer{ + logs: []*Log{&log1, &log2}, + featureService: featureService, + } + loggingService.memoryBuffer = memoryBuffer + table, err := loggingService.GetLogInArrowTable(schema) + offlineStoreConfig := map[string]interface{}{ + "path": ".", + } + fileStore, err := NewFileOfflineStore("test", offlineStoreConfig) + assert.Nil(t, err) + fileStore.FlushToStorage(array.Table(table)) +} diff --git a/go/cmd/server/logging/logging.go b/go/cmd/server/logging/logging.go index 5d5bae36903..58c62f835fb 100644 --- a/go/cmd/server/logging/logging.go +++ b/go/cmd/server/logging/logging.go @@ -6,9 +6,9 @@ import ( "log" "time" - "github.com/apache/arrow/go/arrow" - "github.com/apache/arrow/go/arrow/array" - "github.com/apache/arrow/go/arrow/memory" + "github.com/apache/arrow/go/v8/arrow" + "github.com/apache/arrow/go/v8/arrow/array" + "github.com/apache/arrow/go/v8/arrow/memory" "github.com/feast-dev/feast/go/internal/feast" "github.com/feast-dev/feast/go/protos/feast/serving" "github.com/feast-dev/feast/go/protos/feast/types" @@ -126,7 +126,7 @@ func (s *LoggingService) flushLogsToOfflineStorage(t time.Time) error { return nil } -func (s *LoggingService) getLogInArrowTable(fcoSchema *Schema) (array.Table, error) { +func (s *LoggingService) GetLogInArrowTable(fcoSchema *Schema) (array.Table, error) { // Memory Buffer is a if s.memoryBuffer.featureService == nil { return nil, errors.New("no Feature Service in logging service instantiated") @@ -193,7 +193,6 @@ func (s *LoggingService) getLogInArrowTable(fcoSchema *Schema) (array.Table, err }) columns = append(columns, arrowArray) } - log.Println(columns) schema := arrow.NewSchema( fields, nil, diff --git a/go/cmd/server/logging/logging_test.go b/go/cmd/server/logging/logging_test.go index f4d3b896d56..88fd8710cdb 100644 --- a/go/cmd/server/logging/logging_test.go +++ b/go/cmd/server/logging/logging_test.go @@ -9,8 +9,8 @@ import ( gotypes "github.com/feast-dev/feast/go/types" - "github.com/apache/arrow/go/arrow" - "github.com/apache/arrow/go/arrow/array" + "github.com/apache/arrow/go/v8/arrow" + "github.com/apache/arrow/go/v8/arrow/array" "github.com/feast-dev/feast/go/internal/feast" "github.com/feast-dev/feast/go/protos/feast/serving" "github.com/feast-dev/feast/go/protos/feast/types" @@ -127,7 +127,7 @@ func TestSerializeToArrowTable(t *testing.T) { featureService: featureService, } loggingService.memoryBuffer = memoryBuffer - table, err := loggingService.getLogInArrowTable(schema) + table, err := loggingService.GetLogInArrowTable(schema) defer table.Release() tr := array.NewTableReader(table, -1) expected_schema := map[string]arrow.DataType{ From 0f2aca98187cf69cbf2425e6ab74cdbc50a1bb34 Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Thu, 7 Apr 2022 17:07:39 -0700 Subject: [PATCH 43/97] Working state Signed-off-by: Kevin Zhang --- go.mod | 37 +++++++--- go.sum | 50 +++++++------ go/cmd/server/logging/filelogstorage_test.go | 77 +++++++++++++++++++- go/cmd/server/logging/logging_test.go | 2 - 4 files changed, 132 insertions(+), 34 deletions(-) diff --git a/go.mod b/go.mod index a6375dd11ee..09f92cdb7d3 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,12 @@ module github.com/feast-dev/feast go 1.17 require ( +<<<<<<< HEAD github.com/apache/arrow/go/v8 v8.0.0-20220408212425-58fe60f59289 +======= + github.com/apache/arrow/go/arrow v0.0.0-20211112161151-bc219186db40 + github.com/apache/arrow/go/v8 v8.0.0-20220405223432-9fa34df27eb1 +>>>>>>> 61c3b837 (Working state) github.com/ghodss/yaml v1.0.0 github.com/go-python/gopy v0.4.0 github.com/go-redis/redis/v8 v8.11.4 @@ -12,24 +17,35 @@ require ( github.com/mattn/go-sqlite3 v1.14.12 github.com/spaolacci/murmur3 v1.1.0 github.com/stretchr/testify v1.7.0 +<<<<<<< HEAD google.golang.org/grpc v1.44.0 google.golang.org/protobuf v1.27.1 +======= + github.com/xitongsys/parquet-go v1.6.2 + github.com/xitongsys/parquet-go-source v0.0.0-20220315005136-aec0fe3e777c + google.golang.org/grpc v1.45.0 + google.golang.org/protobuf v1.28.0 +>>>>>>> 61c3b837 (Working state) ) require ( github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c // indirect github.com/andybalholm/brotli v1.0.4 // indirect +<<<<<<< HEAD <<<<<<< HEAD github.com/apache/thrift v0.15.0 // indirect ======= github.com/apache/arrow/go/v8 v8.0.0-20220405223432-9fa34df27eb1 // indirect +======= +>>>>>>> 61c3b837 (Working state) github.com/apache/thrift v0.16.0 // indirect >>>>>>> e0ce4335 (Fix go) github.com/cespare/xxhash/v2 v2.1.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect - github.com/goccy/go-json v0.7.10 // indirect + github.com/goccy/go-json v0.9.6 // indirect github.com/golang/snappy v0.0.4 // indirect +<<<<<<< HEAD github.com/gonuts/commander v0.1.0 // indirect github.com/gonuts/flag v0.1.0 // indirect <<<<<<< HEAD @@ -56,22 +72,25 @@ require ( >>>>>>> e0ce4335 (Fix go) github.com/google/go-cmp v0.5.7 // indirect github.com/klauspost/asmfmt v1.3.1 // indirect +======= + github.com/google/flatbuffers v2.0.6+incompatible // indirect + github.com/klauspost/asmfmt v1.3.2 // indirect +>>>>>>> 61c3b837 (Working state) github.com/klauspost/compress v1.15.1 // indirect - github.com/klauspost/cpuid/v2 v2.0.9 // indirect + github.com/klauspost/cpuid/v2 v2.0.12 // indirect github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 // indirect github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 // indirect - github.com/pierrec/lz4/v4 v4.1.12 // indirect - github.com/pkg/errors v0.9.1 // indirect + github.com/pierrec/lz4/v4 v4.1.14 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/zeebo/xxh3 v1.0.1 // indirect - golang.org/x/exp v0.0.0-20211216164055-b2b84827b756 // indirect + github.com/zeebo/xxh3 v1.0.2 // indirect + golang.org/x/exp v0.0.0-20220407100705-7b9b53b0aca4 // indirect golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3 // indirect - golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd // indirect - golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27 // indirect + golang.org/x/net v0.0.0-20220407224826-aac1ed45d8e3 // indirect + golang.org/x/sys v0.0.0-20220406163625-3f8b81556e12 // indirect golang.org/x/text v0.3.7 // indirect golang.org/x/tools v0.1.10 // indirect golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect - google.golang.org/genproto v0.0.0-20220126215142-9970aeb2e350 // indirect + google.golang.org/genproto v0.0.0-20220407144326-9054f6ed7bac // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c // indirect ) diff --git a/go.sum b/go.sum index 3f87cb76067..eef3b26b787 100644 --- a/go.sum +++ b/go.sum @@ -115,8 +115,9 @@ github.com/go-redis/redis/v8 v8.11.4/go.mod h1:2Z2wHZXdQpCDXEGzqMockDpNyYvi2l4Px github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= -github.com/goccy/go-json v0.7.10 h1:ulhbuNe1JqE68nMRXXTJRrUu0uhouf0VevLINxQq4Ec= github.com/goccy/go-json v0.7.10/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= +github.com/goccy/go-json v0.9.6 h1:5/4CtRQdtsX0sal8fdVhTaiMN01Ri8BExZZ8iRmHQ6E= +github.com/goccy/go-json v0.9.6/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= @@ -145,14 +146,14 @@ github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiu github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/gonuts/commander v0.1.0 h1:EcDTiVw9oAVORFjQOEOuHQqcl6OXMyTgELocTq6zJ0I= github.com/gonuts/commander v0.1.0/go.mod h1:qkb5mSlcWodYgo7vs8ulLnXhfinhZsZcm6+H/z1JjgY= -github.com/gonuts/flag v0.1.0 h1:fqMv/MZ+oNGu0i9gp0/IQ/ZaPIDoAZBOBaJoV7viCWM= github.com/gonuts/flag v0.1.0/go.mod h1:ZTmTGtrSPejTo/SRNhCqwLTmiAgyBdCkLYhHrAoBdz4= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/flatbuffers v2.0.5+incompatible h1:ANsW0idDAXIY+mNHzIHxWRfabV2x5LUEEIIWcwsYgB8= +github.com/google/flatbuffers v1.11.0/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= github.com/google/flatbuffers v2.0.5+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= +github.com/google/flatbuffers v2.0.6+incompatible h1:XHFReMv7nFFusa+CEokzWbzaYocKXI6C7hdU5Kgh9Lw= +github.com/google/flatbuffers v2.0.6+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -212,15 +213,17 @@ github.com/jung-kurt/gofpdf v1.0.0/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+ github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/asmfmt v1.3.1 h1:7xZi1N7s9gTLbqiM8KUv8TLyysavbTRGBT5/ly0bRtw= github.com/klauspost/asmfmt v1.3.1/go.mod h1:AG8TuvYojzulgDAMCnYn50l/5QV3Bs/tp6j0HLHbNSE= +github.com/klauspost/asmfmt v1.3.2 h1:4Ri7ox3EwapiOjCki+hw14RyKk201CN4rzyCJRFLpK4= +github.com/klauspost/asmfmt v1.3.2/go.mod h1:AG8TuvYojzulgDAMCnYn50l/5QV3Bs/tp6j0HLHbNSE= github.com/klauspost/compress v1.9.7/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.13.1/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= github.com/klauspost/compress v1.14.2/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/compress v1.15.1 h1:y9FcTHGyrebwfP0ZZqFiaxTaiDnUrGkJkI+f583BL1A= github.com/klauspost/compress v1.15.1/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= -github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBFYa4= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.0.12 h1:p9dKCg8i4gmOxtv35DvrYoWqYzQrvEVdjQ762Y0OqZE= +github.com/klauspost/cpuid/v2 v2.0.12/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuObKfj5c0PQa7c= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= @@ -297,13 +300,11 @@ github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0 github.com/pierrec/lz4 v2.0.5+incompatible h1:2xWsjqPFWcplujydGg4WmhC/6fZqK42wMM8aXeqhl0I= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pierrec/lz4/v4 v4.1.8/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= -github.com/pierrec/lz4/v4 v4.1.9 h1:xkrjwpOP5xg1k4Nn4GX4a4YFGhscyQL/3EddJ1Xxqm8= -github.com/pierrec/lz4/v4 v4.1.9/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= -github.com/pierrec/lz4/v4 v4.1.12 h1:44l88ehTZAUGW4VlO1QC4zkilL99M6Y9MXNwEs0uzP8= github.com/pierrec/lz4/v4 v4.1.12/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= +github.com/pierrec/lz4/v4 v4.1.14 h1:+fL8AQEZtz/ijeNnpduH0bROTu0O3NZAlPjQxGn8LwE= +github.com/pierrec/lz4/v4 v4.1.14/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/profile v1.2.1/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -368,8 +369,11 @@ github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= -github.com/zeebo/xxh3 v1.0.1 h1:FMSRIbkrLikb/0hZxmltpg84VkqDAT5M8ufXynuhXsI= +github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= +github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= github.com/zeebo/xxh3 v1.0.1/go.mod h1:8VHV24/3AZLn3b6Mlp/KuC33LWH687Wq6EnziEB+rsA= +github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0= +github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= @@ -416,10 +420,9 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/exp v0.0.0-20211028214138-64b4c8e87d1a h1:9kUIHyUjWEuW2MdtaYvvEXTDs2eNylNVAt/ui66QGOg= -golang.org/x/exp v0.0.0-20211028214138-64b4c8e87d1a/go.mod h1:a3o/VtDNHN+dCVLEpzjjUHOzR+Ln3DHX056ZPzoZGGA= -golang.org/x/exp v0.0.0-20211216164055-b2b84827b756 h1:/5Bs7sWi0i3rOVO5KnM55OwugpsD4bRW1zywKoZjbkI= golang.org/x/exp v0.0.0-20211216164055-b2b84827b756/go.mod h1:b9TAUYHmRtqA6klRHApnXMnj+OyLce4yF5cZCUbk2ps= +golang.org/x/exp v0.0.0-20220407100705-7b9b53b0aca4 h1:K3x+yU+fbot38x5bQbU2QqUAVyYLEktdNH2GxZLnM3U= +golang.org/x/exp v0.0.0-20220407100705-7b9b53b0aca4/go.mod h1:lgLbSvA5ygNOMpwM/9anMpWVlVJ7Z+cHWq/eFuinpGE= golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= @@ -446,7 +449,6 @@ golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= golang.org/x/mod v0.6.0-dev.0.20211013180041-c96bc1413d57 h1:LQmS1nU0twXLA96Kt7U9qtHJEbBk3z6Q0V4UXjZkpr4= golang.org/x/mod v0.6.0-dev.0.20211013180041-c96bc1413d57/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.5.1-0.20210830214625-1b1db11ec8f4/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= golang.org/x/mod v0.6.0-dev.0.20211013180041-c96bc1413d57/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3 h1:kQgndtyPBW/JIYERgdxfwMYh3AVStj88WQTlNDi2a+o= @@ -473,9 +475,11 @@ golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= +golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd h1:O7DYs+zxREGLKzKoMQrtrEacpb0ZVXA5rIwylE2Xchk= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220407224826-aac1ed45d8e3 h1:EN5+DfgmRMvRUrMGERW2gQl3Vc+Z7ZMnI/xdEpPSf0c= +golang.org/x/net v0.0.0-20220407224826-aac1ed45d8e3/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -519,8 +523,9 @@ golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27 h1:XDXtA5hveEEV8JB2l7nhMTp3t3cHp9ZpwcdjqyEWLlo= golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220406163625-3f8b81556e12 h1:QyVthZKMsyaQwBTJE04jdNN0Pp5Fn9Qga0mrgxyERQM= +golang.org/x/sys v0.0.0-20220406163625-3f8b81556e12/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -582,10 +587,9 @@ google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98 google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= google.golang.org/genproto v0.0.0-20210630183607-d20f26d13c79/go.mod h1:yiaVoXHpRzHGyxV3o4DktVWY4mSUErTKaeEOq6C3t3U= -google.golang.org/genproto v0.0.0-20220118154757-00ab72f36ad5 h1:zzNejm+EgrbLfDZ6lu9Uud2IVvHySPl8vQzf04laR5Q= -google.golang.org/genproto v0.0.0-20220118154757-00ab72f36ad5/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20220126215142-9970aeb2e350 h1:YxHp5zqIcAShDEvRr5/0rVESVS+njYF68PSdazrNLJo= google.golang.org/genproto v0.0.0-20220126215142-9970aeb2e350/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20220407144326-9054f6ed7bac h1:qSNTkEN+L2mvWcLgJOR+8bdHX9rN/IdU3A1Ghpfb1Rg= +google.golang.org/genproto v0.0.0-20220407144326-9054f6ed7bac/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.0/go.mod h1:chYK+tFQF0nDUGJgXMSgLCQk3phJEuONr2DCgLDdAQM= @@ -600,8 +604,9 @@ google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8 google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= -google.golang.org/grpc v1.44.0 h1:weqSxi/TMs1SqFRMHCtBgXRs8k3X39QIDEZ0pRcttUg= google.golang.org/grpc v1.44.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= +google.golang.org/grpc v1.45.0 h1:NEpgUqV3Z+ZjkqMsxMg11IaDrXY4RY6CQukSGK0uI1M= +google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -613,8 +618,9 @@ google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpAD google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.28.0 h1:w43yiav+6bVFTBQFZX0r7ipe9JQ1QsbMgHwbBziscLw= +google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= diff --git a/go/cmd/server/logging/filelogstorage_test.go b/go/cmd/server/logging/filelogstorage_test.go index a9d155a0763..f807227114c 100644 --- a/go/cmd/server/logging/filelogstorage_test.go +++ b/go/cmd/server/logging/filelogstorage_test.go @@ -1,10 +1,19 @@ package logging import ( + "path/filepath" + + "context" + "os" + "reflect" "testing" "time" + "github.com/apache/arrow/go/v8/arrow" "github.com/apache/arrow/go/v8/arrow/array" + "github.com/apache/arrow/go/v8/arrow/memory" + "github.com/apache/arrow/go/v8/parquet/file" + "github.com/apache/arrow/go/v8/parquet/pqarrow" "github.com/feast-dev/feast/go/protos/feast/serving" "github.com/feast-dev/feast/go/protos/feast/types" "github.com/stretchr/testify/assert" @@ -136,10 +145,76 @@ func TestFlushToStorage(t *testing.T) { } loggingService.memoryBuffer = memoryBuffer table, err := loggingService.GetLogInArrowTable(schema) + defer table.Release() + assert.Nil(t, err) offlineStoreConfig := map[string]interface{}{ "path": ".", } fileStore, err := NewFileOfflineStore("test", offlineStoreConfig) assert.Nil(t, err) - fileStore.FlushToStorage(array.Table(table)) + err = fileStore.FlushToStorage(array.Table(table)) + assert.Nil(t, err) + logPath, err := filepath.Abs(filepath.Join(".", "log.parquet")) + assert.Nil(t, err) + w, err := CreateOrOpenLogFile(logPath) + assert.Nil(t, err) + ctx := context.Background() + + pf, err := file.NewParquetReader(w) + assert.Nil(t, err) + + reader, err := pqarrow.NewFileReader(pf, pqarrow.ArrowReadProperties{}, memory.DefaultAllocator) + assert.Nil(t, err) + + tbl, err := reader.ReadTable(ctx) + assert.Nil(t, err) + tr := array.NewTableReader(tbl, -1) + defer tbl.Release() + expected_schema := map[string]arrow.DataType{ + "driver_id": arrow.PrimitiveTypes.Int64, + "int32": arrow.PrimitiveTypes.Int32, + "double": arrow.PrimitiveTypes.Float64, + "int64": arrow.PrimitiveTypes.Int64, + "float32": arrow.PrimitiveTypes.Float32, + } + + expected_columns := map[string]*types.RepeatedValue{ + "double": { + Val: []*types.Value{{Val: &types.Value_DoubleVal{DoubleVal: 0.97}}, + {Val: &types.Value_DoubleVal{DoubleVal: 8.97}}}}, + "driver_id": { + Val: []*types.Value{{Val: &types.Value_Int64Val{Int64Val: 1001}}, + {Val: &types.Value_Int64Val{Int64Val: 1003}}}}, + "float32": { + Val: []*types.Value{{Val: &types.Value_FloatVal{FloatVal: 0.64}}, + {Val: &types.Value_FloatVal{FloatVal: 1.56}}}}, + "int32": { + Val: []*types.Value{{Val: &types.Value_Int32Val{Int32Val: 55}}, + {Val: &types.Value_Int32Val{Int32Val: 200}}}}, + "int64": { + Val: []*types.Value{{Val: &types.Value_Int64Val{Int64Val: 1000}}, + {Val: &types.Value_Int64Val{Int64Val: 1001}}}}, + } + + defer tr.Release() + for tr.Next() { + rec := tr.Record() + assert.NotNil(t, rec) + for _, field := range rec.Schema().Fields() { + assert.Contains(t, expected_schema, field.Name) + assert.Equal(t, field.Type, expected_schema[field.Name]) + } + values, err := GetProtoFromRecord(rec) + + assert.Nil(t, err) + assert.True(t, reflect.DeepEqual(values, expected_columns)) + } + + err = CleanUpTestLogs(logPath) + assert.Nil(t, err) + +} + +func CleanUpTestLogs(absPath string) error { + return os.Remove(absPath) } diff --git a/go/cmd/server/logging/logging_test.go b/go/cmd/server/logging/logging_test.go index 88fd8710cdb..efa40b1c87d 100644 --- a/go/cmd/server/logging/logging_test.go +++ b/go/cmd/server/logging/logging_test.go @@ -169,8 +169,6 @@ func TestSerializeToArrowTable(t *testing.T) { assert.Nil(t, err) assert.True(t, reflect.DeepEqual(values, expected_columns)) } - - assert.Nil(t, err) } func GetProtoFromRecord(rec array.Record) (map[string]*types.RepeatedValue, error) { r := make(map[string]*types.RepeatedValue) From c631ef85b65ca4b17c58423859bfbf9437b20f6d Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Thu, 7 Apr 2022 19:08:18 -0700 Subject: [PATCH 44/97] Fix Signed-off-by: Kevin Zhang --- go/cmd/server/logging/filelogstorage_test.go | 128 +---------------- go/cmd/server/logging/logging.go | 46 +++---- go/cmd/server/logging/logging_test.go | 136 ++++++++++--------- 3 files changed, 92 insertions(+), 218 deletions(-) diff --git a/go/cmd/server/logging/filelogstorage_test.go b/go/cmd/server/logging/filelogstorage_test.go index f807227114c..9fceae3ab5d 100644 --- a/go/cmd/server/logging/filelogstorage_test.go +++ b/go/cmd/server/logging/filelogstorage_test.go @@ -7,144 +7,18 @@ import ( "os" "reflect" "testing" - "time" "github.com/apache/arrow/go/v8/arrow" "github.com/apache/arrow/go/v8/arrow/array" "github.com/apache/arrow/go/v8/arrow/memory" "github.com/apache/arrow/go/v8/parquet/file" "github.com/apache/arrow/go/v8/parquet/pqarrow" - "github.com/feast-dev/feast/go/protos/feast/serving" "github.com/feast-dev/feast/go/protos/feast/types" "github.com/stretchr/testify/assert" - "google.golang.org/protobuf/types/known/timestamppb" ) -// func TestWriteToLogStorage(t *testing.T) { -// offlineStoreConfig := map[string]interface{}{ -// "path": ".", -// } -// fileStore, err := NewFileOfflineStore("test", offlineStoreConfig) -// assert.Nil(t, err) -// ts := timestamppb.New(time.Now()) -// newLog := Log{ -// EntityName: "driver_id", -// EntityValue: &types.Value{Val: &types.Value_Int64Val{Int64Val: 1001}}, -// FeatureNames: []string{"conv_rate", "acc_rate"}, -// FeatureValues: []*types.Value{{Val: &types.Value_FloatVal{FloatVal: 0.2}}, {Val: &types.Value_FloatVal{FloatVal: 0.5}}}, -// FeatureStatuses: []serving.FieldStatus{serving.FieldStatus_PRESENT, serving.FieldStatus_PRESENT}, -// EventTimestamps: []*timestamppb.Timestamp{ts, ts}, -// } -// newLog2 := Log{ -// EntityName: "driver_id", -// EntityValue: &types.Value{Val: &types.Value_Int64Val{Int64Val: 1003}}, -// FeatureNames: []string{"feature4", "feature5"}, -// FeatureValues: []*types.Value{{Val: &types.Value_FloatVal{FloatVal: 0.3}}, {Val: &types.Value_FloatVal{FloatVal: 0.8}}}, -// FeatureStatuses: []serving.FieldStatus{serving.FieldStatus_PRESENT, serving.FieldStatus_PRESENT}, -// EventTimestamps: []*timestamppb.Timestamp{ts, ts}, -// } -// memoryBuffer := MemoryBuffer{ -// logs: []*Log{&newLog, &newLog2}, -// } - -// err = fileStore.FlushToStorage(&memoryBuffer) -// assert.Nil(t, err) - -// ///read -// fr, err := local.NewLocalFileReader("log.parquet") -// assert.Nil(t, err) - -// pr, err := reader.NewParquetReader(fr, new(ParquetLog), 4) -// if err != nil { -// log.Println("Can't create parquet reader", err) -// return -// } -// num := int(pr.GetNumRows()) -// assert.Equal(t, num, 2) -// logs := make([]ParquetLog, 2) //read 10 rows -// if err = pr.Read(&logs); err != nil { -// log.Println("Read error", err) -// } - -// for i := 0; i < 2; i++ { -// assert.Equal(t, logs[i].EntityName, memoryBuffer.logs[i].EntityName) -// assert.Equal(t, logs[i].EntityValue, memoryBuffer.logs[i].EntityValue.String()) -// assert.True(t, reflect.DeepEqual(logs[i].FeatureNames, memoryBuffer.logs[i].FeatureNames)) -// numValues := len(memoryBuffer.logs[i].FeatureValues) -// assert.Equal(t, numValues, len(logs[i].FeatureValues)) -// assert.Equal(t, numValues, len(logs[i].EventTimestamps)) -// assert.Equal(t, numValues, len(logs[i].FeatureStatuses)) -// for idx := 0; idx < numValues; idx++ { -// assert.Equal(t, logs[i].EventTimestamps[idx], memoryBuffer.logs[i].EventTimestamps[idx].AsTime().UnixNano()/int64(time.Millisecond)) -// if memoryBuffer.logs[i].FeatureStatuses[idx] == serving.FieldStatus_PRESENT { -// assert.True(t, logs[i].FeatureStatuses[idx]) -// } else { -// assert.False(t, logs[i].FeatureStatuses[idx]) -// } -// assert.Equal(t, logs[i].FeatureValues[idx], memoryBuffer.logs[i].FeatureValues[idx].String()) -// } -// } - -// pr.ReadStop() -// fr.Close() - -// err = os.Remove("log.parquet") -// assert.Nil(t, err) -// } - func TestFlushToStorage(t *testing.T) { - featureService, entities, featureViews, odfvs := InitializeFeatureRepoVariablesForTest() - schema, err := GetTypesFromFeatureService(featureService, entities, featureViews, odfvs) - assert.Nil(t, err) - loggingService, err := NewLoggingService(nil, 1, false) - assert.Nil(t, err) - ts := timestamppb.New(time.Now()) - log1 := Log{ - EntityValue: []*types.Value{ - {Val: &types.Value_Int64Val{Int64Val: 1001}}, - }, - FeatureValues: []*types.Value{ - {Val: &types.Value_Int64Val{Int64Val: 1000}}, - {Val: &types.Value_FloatVal{FloatVal: 0.64}}, - {Val: &types.Value_Int32Val{Int32Val: 55}}, - {Val: &types.Value_DoubleVal{DoubleVal: 0.97}}, - }, - FeatureStatuses: []serving.FieldStatus{ - serving.FieldStatus_PRESENT, - serving.FieldStatus_PRESENT, - serving.FieldStatus_PRESENT, - serving.FieldStatus_PRESENT, - }, - EventTimestamps: []*timestamppb.Timestamp{ - ts, ts, ts, ts, - }, - } - log2 := Log{ - EntityValue: []*types.Value{ - {Val: &types.Value_Int64Val{Int64Val: 1003}}, - }, - FeatureValues: []*types.Value{ - {Val: &types.Value_Int64Val{Int64Val: 1001}}, - {Val: &types.Value_FloatVal{FloatVal: 1.56}}, - {Val: &types.Value_Int32Val{Int32Val: 200}}, - {Val: &types.Value_DoubleVal{DoubleVal: 8.97}}, - }, - FeatureStatuses: []serving.FieldStatus{ - serving.FieldStatus_PRESENT, - serving.FieldStatus_PRESENT, - serving.FieldStatus_PRESENT, - serving.FieldStatus_PRESENT, - }, - EventTimestamps: []*timestamppb.Timestamp{ - ts, ts, ts, ts, - }, - } - memoryBuffer := &MemoryBuffer{ - logs: []*Log{&log1, &log2}, - featureService: featureService, - } - loggingService.memoryBuffer = memoryBuffer - table, err := loggingService.GetLogInArrowTable(schema) + table, err := GenerateLogsAndConvertToArrowTable() defer table.Release() assert.Nil(t, err) offlineStoreConfig := map[string]interface{}{ diff --git a/go/cmd/server/logging/logging.go b/go/cmd/server/logging/logging.go index 58c62f835fb..16b3e43ad00 100644 --- a/go/cmd/server/logging/logging.go +++ b/go/cmd/server/logging/logging.go @@ -42,7 +42,6 @@ type LoggingService struct { logChannel chan *Log fs *feast.FeatureStore offlineLogStorage OfflineLogStorage - enableLogging bool } func NewLoggingService(fs *feast.FeatureStore, logChannelCapacity int, enableLogging bool) (*LoggingService, error) { @@ -52,8 +51,7 @@ func NewLoggingService(fs *feast.FeatureStore, logChannelCapacity int, enableLog memoryBuffer: &MemoryBuffer{ logs: make([]*Log, 0), }, - enableLogging: enableLogging, - fs: fs, + fs: fs, } if !enableLogging || fs == nil { loggingService.offlineLogStorage = nil @@ -70,12 +68,12 @@ func NewLoggingService(fs *feast.FeatureStore, logChannelCapacity int, enableLog return loggingService, nil } -func (s *LoggingService) EmitLog(log *Log) error { +func (s *LoggingService) EmitLog(l *Log) error { select { - case s.logChannel <- log: + case s.logChannel <- l: return nil case <-time.After(20 * time.Millisecond): - return fmt.Errorf("could not add to log channel with capacity %d. Current log channel length is %d", cap(s.logChannel), len(s.logChannel)) + return fmt.Errorf("could not add to log channel with capacity %d. Operation timed out. Current log channel length is %d", cap(s.logChannel), len(s.logChannel)) } } @@ -84,22 +82,24 @@ func (s *LoggingService) processLogs() { // TODO(kevjumba): set param so users can configure flushing duration ticker := time.NewTicker(100 * time.Millisecond) defer ticker.Stop() + for { - select { - case t := <-ticker.C: - go s.flushLogsToOfflineStorage(t) - case new_log := <-s.logChannel: - log.Printf("Pushing %s to memory.\n", new_log.FeatureValues) - s.memoryBuffer.logs = append(s.memoryBuffer.logs, new_log) - } + s.ProcessMemoryBuffer(ticker) + } +} + +func (s *LoggingService) ProcessMemoryBuffer(t *time.Ticker) { + select { + case t := <-t.C: + s.flushLogsToOfflineStorage(t) + case new_log := <-s.logChannel: + log.Printf("Pushing %s to memory.\n", new_log.FeatureValues) + s.memoryBuffer.logs = append(s.memoryBuffer.logs, new_log) } } func (s *LoggingService) flushLogsToOfflineStorage(t time.Time) error { log.Printf("Flushing buffer to offline storage with channel length: %d\n at time: "+t.String(), len(s.memoryBuffer.logs)) - if !s.enableLogging { - return nil - } offlineStoreType, ok := getOfflineStoreType(s.fs.GetRepoConfig().OfflineStore) if !ok { return fmt.Errorf("could not get offline storage type for config: %s", s.fs.GetRepoConfig().OfflineStore) @@ -126,19 +126,17 @@ func (s *LoggingService) flushLogsToOfflineStorage(t time.Time) error { return nil } -func (s *LoggingService) GetLogInArrowTable(fcoSchema *Schema) (array.Table, error) { - // Memory Buffer is a - if s.memoryBuffer.featureService == nil { - return nil, errors.New("no Feature Service in logging service instantiated") - } - +// Takes memory buffer of logs in array row and converts them to columnar with generated fcoschema generated by GetFcoSchema +// and writes them to arrow table. +// Returns arrow table that contains all of the logs in columnar format. +func ConvertMemoryBufferToArrowTable(memoryBuffer *MemoryBuffer, fcoSchema *Schema) (array.Table, error) { arrowMemory := memory.NewGoAllocator() columnNameToProtoValueArray := make(map[string][]*types.Value) columnNameToStatus := make(map[string][]int32) columnNameToTimestamp := make(map[string][]int64) entityNameToEntityValues := make(map[string][]*types.Value) - for _, log := range s.memoryBuffer.logs { + for _, log := range memoryBuffer.logs { // TODO(kevjumba) get it from the featureview // Get the entities from the feature view // Grab the corresponding join keys and then process using joinkey map to get the entity key related to the features @@ -197,7 +195,7 @@ func (s *LoggingService) GetLogInArrowTable(fcoSchema *Schema) (array.Table, err fields, nil, ) - result := array.Record(array.NewRecord(schema, columns, int64(len(s.memoryBuffer.logs)))) + result := array.Record(array.NewRecord(schema, columns, int64(len(memoryBuffer.logs)))) // create an arrow table -> write this to parquet. tbl := array.NewTableFromRecords(schema, []array.Record{result}) diff --git a/go/cmd/server/logging/logging_test.go b/go/cmd/server/logging/logging_test.go index efa40b1c87d..e4e0f1a9f29 100644 --- a/go/cmd/server/logging/logging_test.go +++ b/go/cmd/server/logging/logging_test.go @@ -1,9 +1,7 @@ package logging import ( - "path/filepath" "reflect" - "runtime" "testing" "time" @@ -18,20 +16,6 @@ import ( "google.golang.org/protobuf/types/known/timestamppb" ) -// Return absolute path to the test_repo directory regardless of the working directory -func getRepoPath(basePath string) string { - // Get the file path of this source file, regardless of the working directory - if basePath == "" { - _, filename, _, ok := runtime.Caller(0) - if !ok { - panic("couldn't find file path of the test file") - } - return filepath.Join(filename, "..", "..", "feature_repo") - } else { - return filepath.Join(basePath, "feature_repo") - } -} - func TestLoggingChannelTimeout(t *testing.T) { // Pregenerated using `feast init`. loggingService, err := NewLoggingService(nil, 1, false) @@ -76,58 +60,8 @@ func TestSchemaTypeRetrieval(t *testing.T) { } func TestSerializeToArrowTable(t *testing.T) { - featureService, entities, featureViews, odfvs := InitializeFeatureRepoVariablesForTest() - schema, err := GetTypesFromFeatureService(featureService, entities, featureViews, odfvs) + table, err := GenerateLogsAndConvertToArrowTable() assert.Nil(t, err) - loggingService, err := NewLoggingService(nil, 1, false) - assert.Nil(t, err) - ts := timestamppb.New(time.Now()) - log1 := Log{ - EntityValue: []*types.Value{ - {Val: &types.Value_Int64Val{Int64Val: 1001}}, - }, - FeatureValues: []*types.Value{ - {Val: &types.Value_Int64Val{Int64Val: 1000}}, - {Val: &types.Value_FloatVal{FloatVal: 0.64}}, - {Val: &types.Value_Int32Val{Int32Val: 55}}, - {Val: &types.Value_DoubleVal{DoubleVal: 0.97}}, - }, - FeatureStatuses: []serving.FieldStatus{ - serving.FieldStatus_PRESENT, - serving.FieldStatus_PRESENT, - serving.FieldStatus_PRESENT, - serving.FieldStatus_PRESENT, - }, - EventTimestamps: []*timestamppb.Timestamp{ - ts, ts, ts, ts, - }, - } - log2 := Log{ - EntityValue: []*types.Value{ - {Val: &types.Value_Int64Val{Int64Val: 1003}}, - }, - FeatureValues: []*types.Value{ - {Val: &types.Value_Int64Val{Int64Val: 1001}}, - {Val: &types.Value_FloatVal{FloatVal: 1.56}}, - {Val: &types.Value_Int32Val{Int32Val: 200}}, - {Val: &types.Value_DoubleVal{DoubleVal: 8.97}}, - }, - FeatureStatuses: []serving.FieldStatus{ - serving.FieldStatus_PRESENT, - serving.FieldStatus_PRESENT, - serving.FieldStatus_PRESENT, - serving.FieldStatus_PRESENT, - }, - EventTimestamps: []*timestamppb.Timestamp{ - ts, ts, ts, ts, - }, - } - memoryBuffer := &MemoryBuffer{ - logs: []*Log{&log1, &log2}, - featureService: featureService, - } - loggingService.memoryBuffer = memoryBuffer - table, err := loggingService.GetLogInArrowTable(schema) defer table.Release() tr := array.NewTableReader(table, -1) expected_schema := map[string]arrow.DataType{ @@ -170,6 +104,7 @@ func TestSerializeToArrowTable(t *testing.T) { assert.True(t, reflect.DeepEqual(values, expected_columns)) } } + func GetProtoFromRecord(rec array.Record) (map[string]*types.RepeatedValue, error) { r := make(map[string]*types.RepeatedValue) schema := rec.Schema() @@ -234,3 +169,70 @@ func InitializeFeatureRepoVariablesForTest() (*feast.FeatureService, []*feast.En ) return featureService, []*feast.Entity{entity1}, []*feast.FeatureView{featureView1, featureView2}, []*feast.OnDemandFeatureView{} } + +func GenerateLogsAndConvertToArrowTable() (array.Table, error) { + featureService, entities, featureViews, odfvs := InitializeFeatureRepoVariablesForTest() + schema, err := GetTypesFromFeatureService(featureService, entities, featureViews, odfvs) + if err != nil { + return nil, err + } + loggingService, err := NewLoggingService(nil, 2, false) + if err != nil { + return nil, err + } + ts := timestamppb.New(time.Now()) + log1 := Log{ + EntityValue: []*types.Value{ + {Val: &types.Value_Int64Val{Int64Val: 1001}}, + }, + FeatureValues: []*types.Value{ + {Val: &types.Value_Int64Val{Int64Val: 1000}}, + {Val: &types.Value_FloatVal{FloatVal: 0.64}}, + {Val: &types.Value_Int32Val{Int32Val: 55}}, + {Val: &types.Value_DoubleVal{DoubleVal: 0.97}}, + }, + FeatureStatuses: []serving.FieldStatus{ + serving.FieldStatus_PRESENT, + serving.FieldStatus_PRESENT, + serving.FieldStatus_PRESENT, + serving.FieldStatus_PRESENT, + }, + EventTimestamps: []*timestamppb.Timestamp{ + ts, ts, ts, ts, + }, + } + log2 := Log{ + EntityValue: []*types.Value{ + {Val: &types.Value_Int64Val{Int64Val: 1003}}, + }, + FeatureValues: []*types.Value{ + {Val: &types.Value_Int64Val{Int64Val: 1001}}, + {Val: &types.Value_FloatVal{FloatVal: 1.56}}, + {Val: &types.Value_Int32Val{Int32Val: 200}}, + {Val: &types.Value_DoubleVal{DoubleVal: 8.97}}, + }, + FeatureStatuses: []serving.FieldStatus{ + serving.FieldStatus_PRESENT, + serving.FieldStatus_PRESENT, + serving.FieldStatus_PRESENT, + serving.FieldStatus_PRESENT, + }, + EventTimestamps: []*timestamppb.Timestamp{ + ts, ts, ts, ts, + }, + } + + dummyTicker := time.NewTicker(10 * time.Second) + // stop the ticker so that the logs are not flushed to offline storage + dummyTicker.Stop() + loggingService.EmitLog(&log1) + + loggingService.EmitLog(&log2) + loggingService.ProcessMemoryBuffer(dummyTicker) + loggingService.ProcessMemoryBuffer(dummyTicker) + table, err := ConvertMemoryBufferToArrowTable(loggingService.memoryBuffer, schema) + if err != nil { + return nil, err + } + return table, nil +} From b0a2166ef8829440d8e6675208cead126268d45d Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Thu, 7 Apr 2022 19:10:55 -0700 Subject: [PATCH 45/97] fix Signed-off-by: Kevin Zhang --- go/cmd/server/logging/logging.go | 2 +- .../tests/integration/online_store/test_universal_online.py | 4 ---- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/go/cmd/server/logging/logging.go b/go/cmd/server/logging/logging.go index 16b3e43ad00..f55e7947edc 100644 --- a/go/cmd/server/logging/logging.go +++ b/go/cmd/server/logging/logging.go @@ -176,7 +176,7 @@ func ConvertMemoryBufferToArrowTable(memoryBuffer *MemoryBuffer, fcoSchema *Sche columns = append(columns, valArrowArray) } - for featureName, _ := range fcoSchema.FeaturesTypes { + for featureName := range fcoSchema.FeaturesTypes { proto_arr := columnNameToProtoValueArray[featureName] diff --git a/sdk/python/tests/integration/online_store/test_universal_online.py b/sdk/python/tests/integration/online_store/test_universal_online.py index f6906208a7b..0c0ccbfc1b9 100644 --- a/sdk/python/tests/integration/online_store/test_universal_online.py +++ b/sdk/python/tests/integration/online_store/test_universal_online.py @@ -1055,14 +1055,12 @@ def assert_feature_service_entity_mapping_correctness( destinations_df, ): if full_feature_names: - print("asdfasdf") feature_service_online_features_dict = get_online_features_dict( environment=environment, features=feature_service, entity_rows=entity_rows, full_feature_names=full_feature_names, ) - print(feature_service_online_features_dict) feature_service_keys = feature_service_online_features_dict.keys() expected_features = [ @@ -1088,8 +1086,6 @@ def assert_feature_service_entity_mapping_correctness( feature_service_online_features_dict[feature_name][i] == df_features[feature_name] ) - - assert(False) else: # using 2 of the same FeatureView without full_feature_names=True will result in collision with pytest.raises(FeatureNameCollisionError): From 9ed79e82dcbda2937426e981d9626c4fc10b8760 Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Thu, 7 Apr 2022 19:13:23 -0700 Subject: [PATCH 46/97] fix Signed-off-by: Kevin Zhang --- go/cmd/server/server.go | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/go/cmd/server/server.go b/go/cmd/server/server.go index c13c8bafa33..6ccef2b4c1d 100644 --- a/go/cmd/server/server.go +++ b/go/cmd/server/server.go @@ -67,7 +67,6 @@ func (s *servingServiceServer) GetOnlineFeatures(ctx context.Context, request *s }, } // Entities are currently part of the features as a value - entityNames := make([]string, 0) entityValues := make([]*prototypes.Value, 0) for name, values := range request.Entities { resp.Metadata.FeatureNames.Val = append(resp.Metadata.FeatureNames.Val, name) @@ -79,7 +78,6 @@ func (s *servingServiceServer) GetOnlineFeatures(ctx context.Context, request *s resp.Results = append(resp.Results, vec) for _, v := range values.Val { - entityNames = append(entityNames, name) entityValues = append(entityValues, v) vec.Values = append(vec.Values, v) vec.Statuses = append(vec.Statuses, serving.FieldStatus_PRESENT) @@ -100,11 +98,11 @@ func (s *servingServiceServer) GetOnlineFeatures(ctx context.Context, request *s EventTimestamps: vector.Timestamps, }) } - go generateLogs(s, entityNames, entityValues, featureNames, resp.Results, request) + go generateLogs(s, entityValues, featureNames, resp.Results, request) return resp, nil } -func generateLogs(s *servingServiceServer, entityNames []string, entityValues []*prototypes.Value, featureNames []string, results []*serving.GetOnlineFeaturesResponse_FeatureVector, request *serving.GetOnlineFeaturesRequest) error { +func generateLogs(s *servingServiceServer, entityValues []*prototypes.Value, featureNames []string, results []*serving.GetOnlineFeaturesResponse_FeatureVector, request *serving.GetOnlineFeaturesRequest) error { // Add a log with the request context if request.RequestContext != nil && len(request.RequestContext) > 0 { requestContextLog := logging.Log{ @@ -132,9 +130,7 @@ func generateLogs(s *servingServiceServer, entityNames []string, entityValues [] eventTimestampLogRows[row_idx][idx-1] = results[idx].EventTimestamps[row_idx] } newLog := logging.Log{ - EntityName: entityNames, EntityValue: entityValues, - FeatureNames: featureNames, FeatureValues: featureValueLogRows[row_idx], FeatureStatuses: featureStatusLogRows[row_idx], EventTimestamps: eventTimestampLogRows[row_idx], From e472104bba6dd4c9eaf7ea2697919ca966b0af66 Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Fri, 8 Apr 2022 12:37:29 -0700 Subject: [PATCH 47/97] Clean up code a bit Signed-off-by: Kevin Zhang --- go/cmd/server/logging/filelogstorage.go | 52 ------- go/cmd/server/logging/logging.go | 45 +++--- go/cmd/server/logging/logging_test.go | 2 +- go/cmd/server/main.go | 2 +- go/cmd/server/server.go | 14 -- go/cmd/server/server_test.go | 133 +++++++++++------- go/internal/feast/featurestore.go | 56 +++++++- go/internal/feast/registry/registry.go | 18 ++- .../requirements/py3.9-ci-requirements.txt | 2 +- .../online_store/test_universal_online.py | 3 +- 10 files changed, 182 insertions(+), 145 deletions(-) diff --git a/go/cmd/server/logging/filelogstorage.go b/go/cmd/server/logging/filelogstorage.go index 72a187ad6c8..d8906a74ab4 100644 --- a/go/cmd/server/logging/filelogstorage.go +++ b/go/cmd/server/logging/filelogstorage.go @@ -65,58 +65,6 @@ func CreateOrOpenLogFile(absPath string) (*os.File, error) { } } -// func (f *FileLogStorage) FlushToStorage(m *MemoryBuffer) error { -// if len(m.logs) == 0 { -// return nil -// } -// var err error -// w, err := CreateOrOpenLogFile(f.path) -// if err != nil { -// return fmt.Errorf("can't create local file with error: %s", err) -// } -// pw, err := writer.NewParquetWriterFromWriter(w, new(ParquetLog), 4) -// if err != nil { -// return fmt.Errorf("can't create parquet writer with error: %s", err) -// } - -// for _, newLog := range m.logs { -// numValues := len(newLog.FeatureValues) -// if numValues != len(newLog.FeatureStatuses) || numValues != len(newLog.EventTimestamps) { -// return errors.New("length of log arrays do not match") -// } -// statuses := make([]bool, numValues) -// timestampsInMillis := make([]int64, numValues) -// featureValues := make([]string, numValues) -// for idx := 0; idx < numValues; idx++ { -// if newLog.FeatureStatuses[idx] == serving.FieldStatus_PRESENT { -// statuses[idx] = true -// } else { -// statuses[idx] = false -// } -// ts := newLog.EventTimestamps[idx] -// timestampsInMillis[idx] = ts.AsTime().UnixNano() / int64(time.Millisecond) -// featureValues[idx] = newLog.FeatureValues[idx].String() -// } -// newParquetLog := ParquetLog{ -// EntityName: newLog.EntityName, -// EntityValue: newLog.EntityValue.String(), -// FeatureNames: newLog.FeatureNames, -// FeatureValues: featureValues, -// FeatureStatuses: statuses, -// EventTimestamps: timestampsInMillis, -// } -// if err = pw.Write(newParquetLog); err != nil { -// return err -// } -// } -// if err = pw.WriteStop(); err != nil { -// return err -// } -// log.Println("Flushed Log to Parquet File Storage") -// w.Close() -// return nil -// } - func (f *FileLogStorage) FlushToStorage(tbl array.Table) error { w, err := CreateOrOpenLogFile(f.path) var writer io.Writer = w diff --git a/go/cmd/server/logging/logging.go b/go/cmd/server/logging/logging.go index f55e7947edc..10f8a9a45ef 100644 --- a/go/cmd/server/logging/logging.go +++ b/go/cmd/server/logging/logging.go @@ -26,12 +26,6 @@ type Log struct { RequestContext map[string]*types.RepeatedValue } -// driver_id, -// 1003, 1004 -// [acc rate conv rate avg_daily_trips] -// [entityvalues, acc_rate conv_rate avg_daily_trips, acc_ratestatus, conv_rate_status] -// [entityvalues, entity value] - type MemoryBuffer struct { featureService *feast.FeatureService logs []*Log @@ -44,12 +38,17 @@ type LoggingService struct { offlineLogStorage OfflineLogStorage } -func NewLoggingService(fs *feast.FeatureStore, logChannelCapacity int, enableLogging bool) (*LoggingService, error) { +func NewLoggingService(fs *feast.FeatureStore, logChannelCapacity int, featureServiceName string, enableLogging bool) (*LoggingService, error) { // start handler processes? + featureService, err := fs.GetFeatureService(featureServiceName, fs.GetRepoConfig().Project) + if err != nil { + return nil, err + } loggingService := &LoggingService{ logChannel: make(chan *Log, logChannelCapacity), memoryBuffer: &MemoryBuffer{ - logs: make([]*Log, 0), + logs: make([]*Log, 0), + featureService: featureService, }, fs: fs, } @@ -105,19 +104,23 @@ func (s *LoggingService) flushLogsToOfflineStorage(t time.Time) error { return fmt.Errorf("could not get offline storage type for config: %s", s.fs.GetRepoConfig().OfflineStore) } if offlineStoreType == "file" { - - //s.offlineLogStorage.FlushToStorage(s.memoryBuffer) - //Clean memory buffer - // Convert row level logs array in memory buffer to an array table in columnar - // entities, fvs, odfvs, err := s.GetFcos() - // if err != nil { - // return nil, err - // } - // fcoSchema, err := GetTypesFromFeatureService(s.memoryBuffer.featureService, entities, fvs, odfvs) - // if err != nil { - // return nil, err - // } - // table, err := s.getLogInArrowTable(fcoSchema) + entities, featureViews, odfvs, err := s.GetFcos() + if err != nil { + return err + } + schema, err := GetTypesFromFeatureService(s.memoryBuffer.featureService, entities, featureViews, odfvs) + if err != nil { + return err + } + table, err := ConvertMemoryBufferToArrowTable(s.memoryBuffer, schema) + if err != nil { + return err + } + fileStore, err := NewFileOfflineStore(s.fs.GetRepoConfig().Project, s.fs.GetRepoConfig().OfflineStore) + if err != nil { + return err + } + fileStore.FlushToStorage(table) s.memoryBuffer.logs = s.memoryBuffer.logs[:0] } else { // Currently don't support any other offline flushing. diff --git a/go/cmd/server/logging/logging_test.go b/go/cmd/server/logging/logging_test.go index e4e0f1a9f29..44121860b38 100644 --- a/go/cmd/server/logging/logging_test.go +++ b/go/cmd/server/logging/logging_test.go @@ -18,7 +18,7 @@ import ( func TestLoggingChannelTimeout(t *testing.T) { // Pregenerated using `feast init`. - loggingService, err := NewLoggingService(nil, 1, false) + loggingService, err := NewLoggingService(nil, 1, "", false) assert.Nil(t, err) assert.Empty(t, loggingService.memoryBuffer.logs) ts := timestamppb.New(time.Now()) diff --git a/go/cmd/server/main.go b/go/cmd/server/main.go index 34861addae4..90068e03971 100644 --- a/go/cmd/server/main.go +++ b/go/cmd/server/main.go @@ -49,7 +49,7 @@ func main() { log.Fatalln(err) } // Disable logging for now - loggingService, err := logging.NewLoggingService(fs, 1000, false) + loggingService, err := logging.NewLoggingService(fs, 1000, "", false) if err != nil { log.Fatalln(err) } diff --git a/go/cmd/server/server.go b/go/cmd/server/server.go index 6ccef2b4c1d..a8de3fdec56 100644 --- a/go/cmd/server/server.go +++ b/go/cmd/server/server.go @@ -2,7 +2,6 @@ package main import ( "context" - "log" "github.com/feast-dev/feast/go/cmd/server/logging" "github.com/feast-dev/feast/go/internal/feast" @@ -33,19 +32,6 @@ func (s *servingServiceServer) GetFeastServingInfo(ctx context.Context, request // Results contains values including the value of the feature, the event timestamp, and feature status in a columnar format. func (s *servingServiceServer) GetOnlineFeatures(ctx context.Context, request *serving.GetOnlineFeaturesRequest) (*serving.GetOnlineFeaturesResponse, error) { featuresOrService, err := s.fs.ParseFeatures(request.GetKind()) - if featuresOrService.FeatureService != nil { - log.Println("INDSFS") - - log.Println(featuresOrService.FeatureService.Projections[0].Features) - log.Println(featuresOrService.FeatureService.Projections[0].JoinKeyMap) - } - // if featuresOrService.FeatureService != nil { - // if schema, ok := schemaCache[featuresOrService.FeatureService]; !ok { - // generateLogSchemaFromFeatureService(featuresOrService.FeatureService) - // } - - // } - if err != nil { return nil, err } diff --git a/go/cmd/server/server_test.go b/go/cmd/server/server_test.go index 20f64cc8220..1d8b4e4578f 100644 --- a/go/cmd/server/server_test.go +++ b/go/cmd/server/server_test.go @@ -15,13 +15,13 @@ import ( "github.com/feast-dev/feast/go/cmd/server/logging" "github.com/feast-dev/feast/go/internal/feast" "github.com/feast-dev/feast/go/internal/test" + "github.com/feast-dev/feast/go/protos/feast/core" "github.com/feast-dev/feast/go/protos/feast/serving" "github.com/feast-dev/feast/go/protos/feast/types" "github.com/stretchr/testify/assert" - "github.com/xitongsys/parquet-go-source/local" - "github.com/xitongsys/parquet-go/reader" "google.golang.org/grpc" "google.golang.org/grpc/test/bufconn" + "google.golang.org/protobuf/types/known/timestamppb" ) // Return absolute path to the test_repo directory regardless of the working directory @@ -65,7 +65,8 @@ func getClient(ctx context.Context, basePath string, enableLogging bool) (servin if err != nil { panic(err) } - loggingService, err := logging.NewLoggingService(fs, 1000, enableLogging) + generateFeatureService(fs) + loggingService, err := logging.NewLoggingService(fs, 1000, "test_service", enableLogging) if err != nil { panic(err) } @@ -182,60 +183,56 @@ func TestGetOnlineFeaturesSqliteWithLogging(t *testing.T) { {Val: &types.Value_Int64Val{Int64Val: 1005}}, }, } - featureNames := []string{"driver_hourly_stats:conv_rate", "driver_hourly_stats:acc_rate", "driver_hourly_stats:avg_daily_trips"} - expectedEntityValuesResp := []*types.Value{ - {Val: &types.Value_Int64Val{Int64Val: 1001}}, - {Val: &types.Value_Int64Val{Int64Val: 1003}}, - {Val: &types.Value_Int64Val{Int64Val: 1005}}, - } - expectedFeatureNamesResp := []string{"conv_rate", "acc_rate", "avg_daily_trips"} + // featureNames := []string{"driver_hourly_stats:conv_rate", "driver_hourly_stats:acc_rate", "driver_hourly_stats:avg_daily_trips"} + // expectedEntityValuesResp := []*types.Value{ + // {Val: &types.Value_Int64Val{Int64Val: 1001}}, + // {Val: &types.Value_Int64Val{Int64Val: 1003}}, + // {Val: &types.Value_Int64Val{Int64Val: 1005}}, + // } + + // expectedFeatureNamesResp := []string{"conv_rate", "acc_rate", "avg_daily_trips"} request := &serving.GetOnlineFeaturesRequest{ - Kind: &serving.GetOnlineFeaturesRequest_Features{ - Features: &serving.FeatureList{ - Val: featureNames, - }, + Kind: &serving.GetOnlineFeaturesRequest_FeatureService{ + FeatureService: "test_service", }, Entities: entities, } - response, err := client.GetOnlineFeatures(ctx, request) - assert.Nil(t, err) - assert.NotNil(t, response) - // Wait for logger to flush. - // TODO(Change this when we add param for flush duration) - time.Sleep(200 * time.Millisecond) - expectedLogValues, expectedLogStatuses, expectedLogMillis := GetExpectedLogRows(featureNames, response.Results) - // read from parquet log file - fr, err := local.NewLocalFileReader("log.parquet") - assert.Nil(t, err) - - pr, err := reader.NewParquetReader(fr, new(logging.ParquetLog), 4) - if err != nil { - log.Println("Can't create parquet reader", err) - return - } - - num := int(pr.GetNumRows()) - assert.Equal(t, num, 3) - logs := make([]logging.ParquetLog, 3) //read 10 rows - err = pr.Read(&logs) + _, err = client.GetOnlineFeatures(ctx, request) + time.Sleep(1 * time.Second) assert.Nil(t, err) - for i := 0; i < 3; i++ { - assert.Equal(t, logs[i].EntityName, "driver_id") - assert.Equal(t, logs[i].EntityValue, expectedEntityValuesResp[i].String()) - assert.True(t, reflect.DeepEqual(logs[i].FeatureNames, expectedFeatureNamesResp)) - numValues := len(expectedFeatureNamesResp) - assert.Equal(t, numValues, len(logs[i].FeatureValues)) - assert.Equal(t, numValues, len(logs[i].EventTimestamps)) - assert.Equal(t, numValues, len(logs[i].FeatureStatuses)) - assert.True(t, reflect.DeepEqual(logs[i].FeatureValues, expectedLogValues[i])) - assert.True(t, reflect.DeepEqual(logs[i].FeatureStatuses, expectedLogStatuses[i])) - assert.True(t, reflect.DeepEqual(logs[i].EventTimestamps, expectedLogMillis[i])) - } + // assert.NotNil(t, response) + // // Wait for logger to flush. + // // TODO(Change this when we add param for flush duration) + // time.Sleep(200 * time.Millisecond) + // expectedLogValues, expectedLogStatuses, expectedLogMillis := GetExpectedLogRows(featureNames, response.Results) + // // read from parquet log file + // fr, err := local.NewLocalFileReader("log.parquet") + // assert.Nil(t, err) - pr.ReadStop() - fr.Close() + // pr, err := reader.NewParquetReader(fr, new(logging.ParquetLog), 4) + // if err != nil { + // log.Println("Can't create parquet reader", err) + // return + // } + // num := int(pr.GetNumRows()) + // assert.Equal(t, num, 3) + // logs := make([]logging.ParquetLog, 3) //read 10 rows + // err = pr.Read(&logs) + // assert.Nil(t, err) + // for i := 0; i < 3; i++ { + // assert.Equal(t, logs[i].EntityName, "driver_id") + // assert.Equal(t, logs[i].EntityValue, expectedEntityValuesResp[i].String()) + // assert.True(t, reflect.DeepEqual(logs[i].FeatureNames, expectedFeatureNamesResp)) + // numValues := len(expectedFeatureNamesResp) + // assert.Equal(t, numValues, len(logs[i].FeatureValues)) + // assert.Equal(t, numValues, len(logs[i].EventTimestamps)) + // assert.Equal(t, numValues, len(logs[i].FeatureStatuses)) + // assert.True(t, reflect.DeepEqual(logs[i].FeatureValues, expectedLogValues[i])) + // assert.True(t, reflect.DeepEqual(logs[i].FeatureStatuses, expectedLogStatuses[i])) + // assert.True(t, reflect.DeepEqual(logs[i].EventTimestamps, expectedLogMillis[i])) + // } err = os.Remove("log.parquet") assert.Nil(t, err) } @@ -264,3 +261,41 @@ func GetExpectedLogRows(featureNames []string, results []*serving.GetOnlineFeatu return featureValueLogRows, featureStatusLogRows, eventTimestampLogRows } + +func generateFeatureService(fs *feast.FeatureStore) { + f1 := core.FeatureSpecV2{ + Name: "conv_rate", + ValueType: types.ValueType_FLOAT, + } + f2 := core.FeatureSpecV2{ + Name: "acc_rate", + ValueType: types.ValueType_FLOAT, + } + f3 := core.FeatureSpecV2{ + Name: "avg_daily_trips", + ValueType: types.ValueType_INT64, + } + projection1 := core.FeatureViewProjection{ + FeatureViewName: "driver_hourly_stats", + FeatureViewNameAlias: "", + FeatureColumns: []*core.FeatureSpecV2{&f1, &f2, &f3}, + JoinKeyMap: map[string]string{}, + } + featureServiceSpec := &core.FeatureServiceSpec{ + Name: "test_service", + Project: "feature_repo", + Features: []*core.FeatureViewProjection{&projection1}, + } + ts := timestamppb.New(time.Now()) + featureServiceMeta := &core.FeatureServiceMeta{ + CreatedTimestamp: ts, + LastUpdatedTimestamp: ts, + } + featureService := &core.FeatureService{ + Spec: featureServiceSpec, + Meta: featureServiceMeta, + } + registry := fs.Registry() + + registry.CacheFeatureService(featureService) +} diff --git a/go/internal/feast/featurestore.go b/go/internal/feast/featurestore.go index 4154b8c1e5a..93f96ca3366 100644 --- a/go/internal/feast/featurestore.go +++ b/go/internal/feast/featurestore.go @@ -3,6 +3,10 @@ package feast import ( "context" "errors" + "fmt" + "log" + "sort" + "strings" "github.com/apache/arrow/go/v8/arrow/memory" "github.com/feast-dev/feast/go/internal/feast/model" @@ -30,6 +34,56 @@ type Features struct { FeatureService *model.FeatureService } +/* + FeatureVector type represent result of retrieving single feature for multiple rows. + It can be imagined as a column in output dataframe / table. + It contains of feature name, list of values (across all rows), + list of statuses and list of timestamp. All these lists have equal length. + And this length is also equal to number of entity rows received in request. +*/ +type FeatureVector struct { + Name string + Values array.Interface + Statuses []serving.FieldStatus + Timestamps []*timestamppb.Timestamp +} + +type featureViewAndRefs struct { + view *FeatureView + featureRefs []string +} + +func (fs *FeatureStore) Registry() *Registry { + return fs.registry +} + +func (f *featureViewAndRefs) View() *FeatureView { + return f.view +} + +func (f *featureViewAndRefs) FeatureRefs() []string { + return f.featureRefs +} + +/* + We group all features from a single request by entities they attached to. + Thus, we will be able to call online retrieval per entity and not per each feature view. + In this struct we collect all features and views that belongs to a group. + We also store here projected entity keys (only ones that needed to retrieve these features) + and indexes to map result of retrieval into output response. +*/ +type GroupedFeaturesPerEntitySet struct { + // A list of requested feature references of the form featureViewName:featureName that share this entity set + featureNames []string + featureViewNames []string + // full feature references as they supposed to appear in response + aliasedFeatureNames []string + // Entity set as a list of EntityKeys to pass to OnlineRead + entityKeys []*prototypes.EntityKey + // Reversed mapping to project result of retrieval from storage to response + indices [][]int +} + // NewFeatureStore constructs a feature store fat client using the // repo config (contents of feature_store.yaml converted to JSON map). func NewFeatureStore(config *registry.RepoConfig, callback transformation.TransformationCallback) (*FeatureStore, error) { @@ -191,7 +245,7 @@ func (fs *FeatureStore) ParseFeatures(kind interface{}) (*Features, error) { return &Features{FeaturesRefs: featureList.Features.GetVal(), FeatureService: nil}, nil } if featureServiceRequest, ok := kind.(*serving.GetOnlineFeaturesRequest_FeatureService); ok { - featureService, err := fs.registry.GetFeatureService(fs.config.Project, featureServiceRequest.FeatureService) + featureService, err := fs.registry.getFeatureService(fs.config.Project, featureServiceRequest.FeatureService) if err != nil { return nil, err } diff --git a/go/internal/feast/registry/registry.go b/go/internal/feast/registry/registry.go index e47034aaf73..d3844c307a1 100644 --- a/go/internal/feast/registry/registry.go +++ b/go/internal/feast/registry/registry.go @@ -3,11 +3,12 @@ package registry import ( "errors" "fmt" - "github.com/feast-dev/feast/go/internal/feast/model" "net/url" "sync" "time" + "github.com/feast-dev/feast/go/internal/feast/model" + "github.com/feast-dev/feast/go/protos/feast/core" ) @@ -36,6 +37,17 @@ type Registry struct { mu sync.Mutex } +func (r *Registry) CacheFeatureService(featureService *core.FeatureService) { + r.mu.Lock() + defer r.mu.Unlock() + if r.cachedFeatureServices == nil { + r.cachedFeatureServices = make(map[string]map[string]*core.FeatureService) + } + r.cachedFeatureServices[featureService.Spec.Project] = map[string]*core.FeatureService{ + featureService.Spec.Name: featureService, + } +} + func NewRegistry(registryConfig *RegistryConfig, repoPath string) (*Registry, error) { registryStoreType := registryConfig.RegistryStoreType registryPath := registryConfig.Path @@ -94,11 +106,11 @@ func (r *Registry) getRegistryProto() (*core.Registry, error) { if err != nil { return registryProto, err } - r.load(registryProto) + r.Load(registryProto) return registryProto, nil } -func (r *Registry) load(registry *core.Registry) { +func (r *Registry) Load(registry *core.Registry) { r.mu.Lock() defer r.mu.Unlock() r.cachedRegistry = registry diff --git a/sdk/python/requirements/py3.9-ci-requirements.txt b/sdk/python/requirements/py3.9-ci-requirements.txt index 03f64cb404a..cf228b9412b 100644 --- a/sdk/python/requirements/py3.9-ci-requirements.txt +++ b/sdk/python/requirements/py3.9-ci-requirements.txt @@ -118,7 +118,7 @@ colorama==0.4.4 # via feast (setup.py) coverage[toml]==6.3.2 # via pytest-cov -cryptography==3.4.8 +cryptography==3.3.2 # via # adal # azure-identity diff --git a/sdk/python/tests/integration/online_store/test_universal_online.py b/sdk/python/tests/integration/online_store/test_universal_online.py index 0c0ccbfc1b9..6762f01a1f3 100644 --- a/sdk/python/tests/integration/online_store/test_universal_online.py +++ b/sdk/python/tests/integration/online_store/test_universal_online.py @@ -703,8 +703,7 @@ def test_online_retrieval_with_go_server( fs = go_environment.feature_store entities, datasets, data_sources = go_data_sources feature_views = construct_universal_feature_views(data_sources, with_odfv=False) - print("ASdfasdf") - print(feature_views.location) + feature_service_entity_mapping = FeatureService( name="entity_mapping", features=[ From 629be1825d815a177387b85a11e0903c366f623b Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Fri, 8 Apr 2022 15:44:57 -0700 Subject: [PATCH 48/97] Fixes Signed-off-by: Kevin Zhang --- go/cmd/server/logging/filelogstorage_test.go | 11 +- go/cmd/server/logging/logging.go | 16 ++- go/cmd/server/logging/logging_test.go | 17 +-- go/cmd/server/server.go | 35 +++--- go/cmd/server/server_test.go | 112 +++++++++--------- go/internal/feast/featurestore.go | 1 - go/internal/test/go_integration_test_utils.go | 24 +++- 7 files changed, 113 insertions(+), 103 deletions(-) diff --git a/go/cmd/server/logging/filelogstorage_test.go b/go/cmd/server/logging/filelogstorage_test.go index 9fceae3ab5d..b88d1ad9f34 100644 --- a/go/cmd/server/logging/filelogstorage_test.go +++ b/go/cmd/server/logging/filelogstorage_test.go @@ -4,7 +4,6 @@ import ( "path/filepath" "context" - "os" "reflect" "testing" @@ -13,6 +12,7 @@ import ( "github.com/apache/arrow/go/v8/arrow/memory" "github.com/apache/arrow/go/v8/parquet/file" "github.com/apache/arrow/go/v8/parquet/pqarrow" + "github.com/feast-dev/feast/go/internal/test" "github.com/feast-dev/feast/go/protos/feast/types" "github.com/stretchr/testify/assert" ) @@ -78,17 +78,12 @@ func TestFlushToStorage(t *testing.T) { assert.Contains(t, expected_schema, field.Name) assert.Equal(t, field.Type, expected_schema[field.Name]) } - values, err := GetProtoFromRecord(rec) + values, err := test.GetProtoFromRecord(rec) assert.Nil(t, err) assert.True(t, reflect.DeepEqual(values, expected_columns)) } - err = CleanUpTestLogs(logPath) + err = test.CleanUpLogs(logPath) assert.Nil(t, err) - -} - -func CleanUpTestLogs(absPath string) error { - return os.Remove(absPath) } diff --git a/go/cmd/server/logging/logging.go b/go/cmd/server/logging/logging.go index 10f8a9a45ef..a1f7b27a829 100644 --- a/go/cmd/server/logging/logging.go +++ b/go/cmd/server/logging/logging.go @@ -40,6 +40,7 @@ type LoggingService struct { func NewLoggingService(fs *feast.FeatureStore, logChannelCapacity int, featureServiceName string, enableLogging bool) (*LoggingService, error) { // start handler processes? + var featureService *feast.FeatureService = nil featureService, err := fs.GetFeatureService(featureServiceName, fs.GetRepoConfig().Project) if err != nil { return nil, err @@ -139,17 +140,19 @@ func ConvertMemoryBufferToArrowTable(memoryBuffer *MemoryBuffer, fcoSchema *Sche columnNameToStatus := make(map[string][]int32) columnNameToTimestamp := make(map[string][]int64) entityNameToEntityValues := make(map[string][]*types.Value) - for _, log := range memoryBuffer.logs { + for _, l := range memoryBuffer.logs { // TODO(kevjumba) get it from the featureview // Get the entities from the feature view // Grab the corresponding join keys and then process using joinkey map to get the entity key related to the features // For each entity key create a new column // populate entitynametoentityvalues map + log.Println(fcoSchema.EntityTypes) + log.Println(l.EntityValue) for entityName, idAndType := range fcoSchema.EntityTypes { if _, ok := entityNameToEntityValues[entityName]; !ok { entityNameToEntityValues[entityName] = make([]*types.Value, 0) } - entityNameToEntityValues[entityName] = append(entityNameToEntityValues[entityName], log.EntityValue[idAndType.index]) + entityNameToEntityValues[entityName] = append(entityNameToEntityValues[entityName], l.EntityValue[idAndType.index]) } for featureName, idAndType := range fcoSchema.FeaturesTypes { @@ -159,9 +162,9 @@ func ConvertMemoryBufferToArrowTable(memoryBuffer *MemoryBuffer, fcoSchema *Sche columnNameToStatus[featureName] = make([]int32, 0) columnNameToTimestamp[featureName] = make([]int64, 0) } - columnNameToProtoValueArray[featureName] = append(columnNameToProtoValueArray[featureName], log.FeatureValues[idAndType.index]) - columnNameToStatus[featureName] = append(columnNameToStatus[featureName], int32(log.FeatureStatuses[idAndType.index])) - columnNameToTimestamp[featureName] = append(columnNameToTimestamp[featureName], log.EventTimestamps[idAndType.index].AsTime().UnixNano()/int64(time.Millisecond)) + columnNameToProtoValueArray[featureName] = append(columnNameToProtoValueArray[featureName], l.FeatureValues[idAndType.index]) + columnNameToStatus[featureName] = append(columnNameToStatus[featureName], int32(l.FeatureStatuses[idAndType.index])) + columnNameToTimestamp[featureName] = append(columnNameToTimestamp[featureName], l.EventTimestamps[idAndType.index].AsTime().UnixNano()/int64(time.Millisecond)) } } @@ -207,6 +210,7 @@ func ConvertMemoryBufferToArrowTable(memoryBuffer *MemoryBuffer, fcoSchema *Sche } type Schema struct { + Entities []string EntityTypes map[string]*IndexAndType FeaturesTypes map[string]*IndexAndType RequestDataTypes map[string]*IndexAndType @@ -282,7 +286,7 @@ func (s *LoggingService) GetFcos() ([]*feast.Entity, []*feast.FeatureView, []*fe if err != nil { return nil, nil, nil, err } - entities, err := s.fs.ListEntities(false) + entities, err := s.fs.ListEntities(true) if err != nil { return nil, nil, nil, err } diff --git a/go/cmd/server/logging/logging_test.go b/go/cmd/server/logging/logging_test.go index 44121860b38..fb9a7776288 100644 --- a/go/cmd/server/logging/logging_test.go +++ b/go/cmd/server/logging/logging_test.go @@ -5,8 +5,6 @@ import ( "testing" "time" - gotypes "github.com/feast-dev/feast/go/types" - "github.com/apache/arrow/go/v8/arrow" "github.com/apache/arrow/go/v8/arrow/array" "github.com/feast-dev/feast/go/internal/feast" @@ -105,19 +103,6 @@ func TestSerializeToArrowTable(t *testing.T) { } } -func GetProtoFromRecord(rec array.Record) (map[string]*types.RepeatedValue, error) { - r := make(map[string]*types.RepeatedValue) - schema := rec.Schema() - for idx, column := range rec.Columns() { - field := schema.Field(idx) - values, err := gotypes.ArrowValuesToProtoValues(column) - if err != nil { - return nil, err - } - r[field.Name] = &types.RepeatedValue{Val: values} - } - return r, nil -} func InitializeFeatureRepoVariablesForTest() (*feast.FeatureService, []*feast.Entity, []*feast.FeatureView, []*feast.OnDemandFeatureView) { f1 := feast.NewFeature( "int64", @@ -176,7 +161,7 @@ func GenerateLogsAndConvertToArrowTable() (array.Table, error) { if err != nil { return nil, err } - loggingService, err := NewLoggingService(nil, 2, false) + loggingService, err := NewLoggingService(nil, 2, "", false) if err != nil { return nil, err } diff --git a/go/cmd/server/server.go b/go/cmd/server/server.go index a8de3fdec56..fd8909d94cf 100644 --- a/go/cmd/server/server.go +++ b/go/cmd/server/server.go @@ -53,7 +53,7 @@ func (s *servingServiceServer) GetOnlineFeatures(ctx context.Context, request *s }, } // Entities are currently part of the features as a value - entityValues := make([]*prototypes.Value, 0) + entityValues := make(map[string][]*prototypes.Value, 0) for name, values := range request.Entities { resp.Metadata.FeatureNames.Val = append(resp.Metadata.FeatureNames.Val, name) vec := &serving.GetOnlineFeaturesResponse_FeatureVector{ @@ -62,13 +62,14 @@ func (s *servingServiceServer) GetOnlineFeatures(ctx context.Context, request *s EventTimestamps: make([]*timestamp.Timestamp, 0), } resp.Results = append(resp.Results, vec) - + valueSlice := make([]*prototypes.Value, 0) for _, v := range values.Val { - entityValues = append(entityValues, v) + valueSlice = append(valueSlice, v) vec.Values = append(vec.Values, v) vec.Statuses = append(vec.Statuses, serving.FieldStatus_PRESENT) vec.EventTimestamps = append(vec.EventTimestamps, ×tamp.Timestamp{}) } + entityValues[name] = valueSlice } featureNames := make([]string, len(featureVectors)) for idx, vector := range featureVectors { @@ -84,24 +85,26 @@ func (s *servingServiceServer) GetOnlineFeatures(ctx context.Context, request *s EventTimestamps: vector.Timestamps, }) } - go generateLogs(s, entityValues, featureNames, resp.Results, request) + go generateLogs(s, entityValues, featureNames, resp.Results, request.RequestContext) return resp, nil } -func generateLogs(s *servingServiceServer, entityValues []*prototypes.Value, featureNames []string, results []*serving.GetOnlineFeaturesResponse_FeatureVector, request *serving.GetOnlineFeaturesRequest) error { +func generateLogs(s *servingServiceServer, entities map[string][]*prototypes.Value, featureNames []string, features []*serving.GetOnlineFeaturesResponse_FeatureVector, requestData map[string]*prototypes.RepeatedValue) error { // Add a log with the request context - if request.RequestContext != nil && len(request.RequestContext) > 0 { + if requestData != nil && len(requestData) > 0 { requestContextLog := logging.Log{ - RequestContext: request.RequestContext, + RequestContext: requestData, } s.loggingService.EmitLog(&requestContextLog) } - if len(results) <= 0 { + //schema := getSchemaFromFeatureService(featureService) + //featuresByName := make(map[string]*serving.GetOnlineFeaturesResponse_FeatureVector) + if len(features) <= 0 { return nil } numFeatures := len(featureNames) - numRows := len(results[0].Values) + numRows := len(features[0].Values) featureValueLogRows := make([][]*prototypes.Value, numRows) featureStatusLogRows := make([][]serving.FieldStatus, numRows) eventTimestampLogRows := make([][]*timestamppb.Timestamp, numRows) @@ -110,13 +113,17 @@ func generateLogs(s *servingServiceServer, entityValues []*prototypes.Value, fea featureValueLogRows[row_idx] = make([]*prototypes.Value, numFeatures) featureStatusLogRows[row_idx] = make([]serving.FieldStatus, numFeatures) eventTimestampLogRows[row_idx] = make([]*timestamppb.Timestamp, numFeatures) - for idx := 1; idx < len(results); idx++ { - featureValueLogRows[row_idx][idx-1] = results[idx].Values[row_idx] - featureStatusLogRows[row_idx][idx-1] = results[idx].Statuses[row_idx] - eventTimestampLogRows[row_idx][idx-1] = results[idx].EventTimestamps[row_idx] + for idx := 1; idx < len(features); idx++ { + featureValueLogRows[row_idx][idx-1] = features[idx].Values[row_idx] + featureStatusLogRows[row_idx][idx-1] = features[idx].Statuses[row_idx] + eventTimestampLogRows[row_idx][idx-1] = features[idx].EventTimestamps[row_idx] + } + entityRow := make([]*prototypes.Value, 0) + for _, val := range entities { + entityRow = append(entityRow, val[row_idx]) } newLog := logging.Log{ - EntityValue: entityValues, + EntityValue: entityRow, FeatureValues: featureValueLogRows[row_idx], FeatureStatuses: featureStatusLogRows[row_idx], EventTimestamps: eventTimestampLogRows[row_idx], diff --git a/go/cmd/server/server_test.go b/go/cmd/server/server_test.go index 1d8b4e4578f..9ef3cdf6bb3 100644 --- a/go/cmd/server/server_test.go +++ b/go/cmd/server/server_test.go @@ -5,13 +5,16 @@ import ( "github.com/feast-dev/feast/go/internal/feast/registry" "log" "net" - "os" "path/filepath" "reflect" "runtime" "testing" "time" + "github.com/apache/arrow/go/v8/arrow/array" + "github.com/apache/arrow/go/v8/arrow/memory" + "github.com/apache/arrow/go/v8/parquet/file" + "github.com/apache/arrow/go/v8/parquet/pqarrow" "github.com/feast-dev/feast/go/cmd/server/logging" "github.com/feast-dev/feast/go/internal/feast" "github.com/feast-dev/feast/go/internal/test" @@ -184,78 +187,73 @@ func TestGetOnlineFeaturesSqliteWithLogging(t *testing.T) { }, } - // featureNames := []string{"driver_hourly_stats:conv_rate", "driver_hourly_stats:acc_rate", "driver_hourly_stats:avg_daily_trips"} - // expectedEntityValuesResp := []*types.Value{ - // {Val: &types.Value_Int64Val{Int64Val: 1001}}, - // {Val: &types.Value_Int64Val{Int64Val: 1003}}, - // {Val: &types.Value_Int64Val{Int64Val: 1005}}, - // } - - // expectedFeatureNamesResp := []string{"conv_rate", "acc_rate", "avg_daily_trips"} request := &serving.GetOnlineFeaturesRequest{ Kind: &serving.GetOnlineFeaturesRequest_FeatureService{ FeatureService: "test_service", }, Entities: entities, } - _, err = client.GetOnlineFeatures(ctx, request) + response, err := client.GetOnlineFeatures(ctx, request) time.Sleep(1 * time.Second) assert.Nil(t, err) - // assert.NotNil(t, response) + assert.NotNil(t, response) // // Wait for logger to flush. - // // TODO(Change this when we add param for flush duration) - // time.Sleep(200 * time.Millisecond) - // expectedLogValues, expectedLogStatuses, expectedLogMillis := GetExpectedLogRows(featureNames, response.Results) - // // read from parquet log file - // fr, err := local.NewLocalFileReader("log.parquet") - // assert.Nil(t, err) + // Get the featurenames without the entity order + featureNames := response.Metadata.FeatureNames.Val[len(request.Entities):] + expectedLogValues, _, _ := GetExpectedLogRows(featureNames, response.Results) + expectedLogValues["driver_id"] = entities["driver_id"] + logPath, err := filepath.Abs(filepath.Join(".", "log.parquet")) + assert.Nil(t, err) + w, err := logging.CreateOrOpenLogFile(logPath) + assert.Nil(t, err) - // pr, err := reader.NewParquetReader(fr, new(logging.ParquetLog), 4) - // if err != nil { - // log.Println("Can't create parquet reader", err) - // return - // } + pf, err := file.NewParquetReader(w) + assert.Nil(t, err) - // num := int(pr.GetNumRows()) - // assert.Equal(t, num, 3) - // logs := make([]logging.ParquetLog, 3) //read 10 rows - // err = pr.Read(&logs) - // assert.Nil(t, err) - // for i := 0; i < 3; i++ { - // assert.Equal(t, logs[i].EntityName, "driver_id") - // assert.Equal(t, logs[i].EntityValue, expectedEntityValuesResp[i].String()) - // assert.True(t, reflect.DeepEqual(logs[i].FeatureNames, expectedFeatureNamesResp)) - // numValues := len(expectedFeatureNamesResp) - // assert.Equal(t, numValues, len(logs[i].FeatureValues)) - // assert.Equal(t, numValues, len(logs[i].EventTimestamps)) - // assert.Equal(t, numValues, len(logs[i].FeatureStatuses)) - // assert.True(t, reflect.DeepEqual(logs[i].FeatureValues, expectedLogValues[i])) - // assert.True(t, reflect.DeepEqual(logs[i].FeatureStatuses, expectedLogStatuses[i])) - // assert.True(t, reflect.DeepEqual(logs[i].EventTimestamps, expectedLogMillis[i])) - // } - err = os.Remove("log.parquet") + reader, err := pqarrow.NewFileReader(pf, pqarrow.ArrowReadProperties{}, memory.DefaultAllocator) + assert.Nil(t, err) + + tbl, err := reader.ReadTable(ctx) + assert.Nil(t, err) + tr := array.NewTableReader(tbl, -1) + defer tbl.Release() + defer tr.Release() + for tr.Next() { + rec := tr.Record() + assert.NotNil(t, rec) + values, err := test.GetProtoFromRecord(rec) + + assert.Nil(t, err) + assert.Equal(t, len(values), len(expectedLogValues)) + for name, val := range values { + assert.Equal(t, len(val.Val), len(expectedLogValues[name].Val)) + for idx, featureVal := range val.Val { + assert.Equal(t, featureVal.Val, expectedLogValues[name].Val[idx].Val) + } + } + } + err = test.CleanUpLogs(logPath) assert.Nil(t, err) } -func GetExpectedLogRows(featureNames []string, results []*serving.GetOnlineFeaturesResponse_FeatureVector) ([][]string, [][]bool, [][]int64) { +func GetExpectedLogRows(featureNames []string, results []*serving.GetOnlineFeaturesResponse_FeatureVector) (map[string]*types.RepeatedValue, [][]int32, [][]int64) { numFeatures := len(featureNames) numRows := len(results[0].Values) - featureValueLogRows := make([][]string, numRows) - featureStatusLogRows := make([][]bool, numRows) + featureValueLogRows := make(map[string]*types.RepeatedValue) + featureStatusLogRows := make([][]int32, numRows) eventTimestampLogRows := make([][]int64, numRows) - - for row_idx := 0; row_idx < numRows; row_idx++ { - featureValueLogRows[row_idx] = make([]string, numFeatures) - featureStatusLogRows[row_idx] = make([]bool, numFeatures) - eventTimestampLogRows[row_idx] = make([]int64, numFeatures) - for idx := 1; idx < len(results); idx++ { - featureValueLogRows[row_idx][idx-1] = results[idx].Values[row_idx].String() - if results[idx].Statuses[row_idx] == serving.FieldStatus_PRESENT { - featureStatusLogRows[row_idx][idx-1] = true - } else { - featureStatusLogRows[row_idx][idx-1] = false - } + for idx := 1; idx < len(results); idx++ { + valArray := make([]*types.Value, 0) + for row_idx := 0; row_idx < numRows; row_idx++ { + featureStatusLogRows[row_idx] = make([]int32, numFeatures) + eventTimestampLogRows[row_idx] = make([]int64, numFeatures) + valArray = append(valArray, results[idx].Values[row_idx]) + featureStatusLogRows[row_idx][idx-1] = int32(serving.FieldStatus_PRESENT) eventTimestampLogRows[row_idx][idx-1] = results[idx].EventTimestamps[row_idx].AsTime().UnixNano() / int64(time.Millisecond) + + } + featureValueLogRows[featureNames[idx-1]] = &types.RepeatedValue{ + Val: valArray, } } @@ -264,11 +262,11 @@ func GetExpectedLogRows(featureNames []string, results []*serving.GetOnlineFeatu func generateFeatureService(fs *feast.FeatureStore) { f1 := core.FeatureSpecV2{ - Name: "conv_rate", + Name: "acc_rate", ValueType: types.ValueType_FLOAT, } f2 := core.FeatureSpecV2{ - Name: "acc_rate", + Name: "conv_rate", ValueType: types.ValueType_FLOAT, } f3 := core.FeatureSpecV2{ diff --git a/go/internal/feast/featurestore.go b/go/internal/feast/featurestore.go index 93f96ca3366..2252d293fff 100644 --- a/go/internal/feast/featurestore.go +++ b/go/internal/feast/featurestore.go @@ -4,7 +4,6 @@ import ( "context" "errors" "fmt" - "log" "sort" "strings" diff --git a/go/internal/test/go_integration_test_utils.go b/go/internal/test/go_integration_test_utils.go index 8e8b0f9a0cf..c95ca034887 100644 --- a/go/internal/test/go_integration_test_utils.go +++ b/go/internal/test/go_integration_test_utils.go @@ -3,7 +3,7 @@ package test import ( "context" "fmt" - "github.com/apache/arrow/go/v8/arrow/array" + "github.com/apache/arrow/go/v8/arrow/memory" "github.com/apache/arrow/go/v8/arrow" @@ -14,6 +14,10 @@ import ( "os/exec" "path/filepath" "time" + + "github.com/apache/arrow/go/v8/arrow/array" + "github.com/feast-dev/feast/go/protos/feast/types" + gotypes "github.com/feast-dev/feast/go/types" ) type Row struct { @@ -128,3 +132,21 @@ func CleanUpRepo(basePath string) error { } return nil } + +func GetProtoFromRecord(rec array.Record) (map[string]*types.RepeatedValue, error) { + r := make(map[string]*types.RepeatedValue) + schema := rec.Schema() + for idx, column := range rec.Columns() { + field := schema.Field(idx) + values, err := gotypes.ArrowValuesToProtoValues(column) + if err != nil { + return nil, err + } + r[field.Name] = &types.RepeatedValue{Val: values} + } + return r, nil +} + +func CleanUpLogs(absPath string) error { + return os.Remove(absPath) +} From 8a789fbce5ddd0d8477cb72a20170993e21e9e4b Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Fri, 8 Apr 2022 15:53:06 -0700 Subject: [PATCH 49/97] Fix Signed-off-by: Kevin Zhang --- go/cmd/server/logging/logging.go | 7 ------- go/cmd/server/server.go | 4 +--- go/embedded/online_features.go | 1 + go/internal/feast/featurestore.go | 2 +- 4 files changed, 3 insertions(+), 11 deletions(-) diff --git a/go/cmd/server/logging/logging.go b/go/cmd/server/logging/logging.go index a1f7b27a829..8399ebc8f5e 100644 --- a/go/cmd/server/logging/logging.go +++ b/go/cmd/server/logging/logging.go @@ -141,13 +141,6 @@ func ConvertMemoryBufferToArrowTable(memoryBuffer *MemoryBuffer, fcoSchema *Sche columnNameToTimestamp := make(map[string][]int64) entityNameToEntityValues := make(map[string][]*types.Value) for _, l := range memoryBuffer.logs { - // TODO(kevjumba) get it from the featureview - // Get the entities from the feature view - // Grab the corresponding join keys and then process using joinkey map to get the entity key related to the features - // For each entity key create a new column - // populate entitynametoentityvalues map - log.Println(fcoSchema.EntityTypes) - log.Println(l.EntityValue) for entityName, idAndType := range fcoSchema.EntityTypes { if _, ok := entityNameToEntityValues[entityName]; !ok { entityNameToEntityValues[entityName] = make([]*types.Value, 0) diff --git a/go/cmd/server/server.go b/go/cmd/server/server.go index fd8909d94cf..16f50afda61 100644 --- a/go/cmd/server/server.go +++ b/go/cmd/server/server.go @@ -91,15 +91,13 @@ func (s *servingServiceServer) GetOnlineFeatures(ctx context.Context, request *s func generateLogs(s *servingServiceServer, entities map[string][]*prototypes.Value, featureNames []string, features []*serving.GetOnlineFeaturesResponse_FeatureVector, requestData map[string]*prototypes.RepeatedValue) error { // Add a log with the request context - if requestData != nil && len(requestData) > 0 { + if len(requestData) > 0 { requestContextLog := logging.Log{ RequestContext: requestData, } s.loggingService.EmitLog(&requestContextLog) } - //schema := getSchemaFromFeatureService(featureService) - //featuresByName := make(map[string]*serving.GetOnlineFeaturesResponse_FeatureVector) if len(features) <= 0 { return nil } diff --git a/go/embedded/online_features.go b/go/embedded/online_features.go index 26eab4f48d0..104d171cac6 100644 --- a/go/embedded/online_features.go +++ b/go/embedded/online_features.go @@ -142,6 +142,7 @@ func (s *OnlineFeatureService) GetOnlineFeatures( if featureServiceName != "" { featureService, err = s.fs.GetFeatureService(featureServiceName) } + resp, err := s.fs.GetOnlineFeatures( context.Background(), featureRefs, diff --git a/go/internal/feast/featurestore.go b/go/internal/feast/featurestore.go index 2252d293fff..b1c0a136521 100644 --- a/go/internal/feast/featurestore.go +++ b/go/internal/feast/featurestore.go @@ -244,7 +244,7 @@ func (fs *FeatureStore) ParseFeatures(kind interface{}) (*Features, error) { return &Features{FeaturesRefs: featureList.Features.GetVal(), FeatureService: nil}, nil } if featureServiceRequest, ok := kind.(*serving.GetOnlineFeaturesRequest_FeatureService); ok { - featureService, err := fs.registry.getFeatureService(fs.config.Project, featureServiceRequest.FeatureService) + featureService, err := fs.registry.GetFeatureService(fs.config.Project, featureServiceRequest.FeatureService) if err != nil { return nil, err } From 6357efd741598f78c2cf073c24024f4f144671f2 Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Fri, 8 Apr 2022 16:04:19 -0700 Subject: [PATCH 50/97] Fix tests Signed-off-by: Kevin Zhang --- go/cmd/server/logging/logging.go | 13 ++++++++++--- go/cmd/server/logging/logging_test.go | 3 ++- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/go/cmd/server/logging/logging.go b/go/cmd/server/logging/logging.go index 8399ebc8f5e..bffb118eeb2 100644 --- a/go/cmd/server/logging/logging.go +++ b/go/cmd/server/logging/logging.go @@ -41,10 +41,14 @@ type LoggingService struct { func NewLoggingService(fs *feast.FeatureStore, logChannelCapacity int, featureServiceName string, enableLogging bool) (*LoggingService, error) { // start handler processes? var featureService *feast.FeatureService = nil - featureService, err := fs.GetFeatureService(featureServiceName, fs.GetRepoConfig().Project) - if err != nil { - return nil, err + var err error + if enableLogging { + featureService, err = fs.GetFeatureService(featureServiceName, fs.GetRepoConfig().Project) + if err != nil { + return nil, err + } } + loggingService := &LoggingService{ logChannel: make(chan *Log, logChannelCapacity), memoryBuffer: &MemoryBuffer{ @@ -53,6 +57,7 @@ func NewLoggingService(fs *feast.FeatureStore, logChannelCapacity int, featureSe }, fs: fs, } + if !enableLogging || fs == nil { loggingService.offlineLogStorage = nil } else { @@ -141,6 +146,8 @@ func ConvertMemoryBufferToArrowTable(memoryBuffer *MemoryBuffer, fcoSchema *Sche columnNameToTimestamp := make(map[string][]int64) entityNameToEntityValues := make(map[string][]*types.Value) for _, l := range memoryBuffer.logs { + // EntityTypes maps an entity name to the specific type and also which index in the entityValues array it is + // e.g if an Entity Key is {driver_id, customer_id}, then the driver_id entitytype would be dtype=int64, index=0. for entityName, idAndType := range fcoSchema.EntityTypes { if _, ok := entityNameToEntityValues[entityName]; !ok { entityNameToEntityValues[entityName] = make([]*types.Value, 0) diff --git a/go/cmd/server/logging/logging_test.go b/go/cmd/server/logging/logging_test.go index fb9a7776288..323020ccf3d 100644 --- a/go/cmd/server/logging/logging_test.go +++ b/go/cmd/server/logging/logging_test.go @@ -8,6 +8,7 @@ import ( "github.com/apache/arrow/go/v8/arrow" "github.com/apache/arrow/go/v8/arrow/array" "github.com/feast-dev/feast/go/internal/feast" + "github.com/feast-dev/feast/go/internal/test" "github.com/feast-dev/feast/go/protos/feast/serving" "github.com/feast-dev/feast/go/protos/feast/types" "github.com/stretchr/testify/assert" @@ -96,7 +97,7 @@ func TestSerializeToArrowTable(t *testing.T) { assert.Contains(t, expected_schema, field.Name) assert.Equal(t, field.Type, expected_schema[field.Name]) } - values, err := GetProtoFromRecord(rec) + values, err := test.GetProtoFromRecord(rec) assert.Nil(t, err) assert.True(t, reflect.DeepEqual(values, expected_columns)) From 3168473451fa9111e923ebd66fc62c15f14023dc Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Fri, 8 Apr 2022 16:49:12 -0700 Subject: [PATCH 51/97] Fix Signed-off-by: Kevin Zhang --- go/cmd/server/logging/logging.go | 16 ++++++++++++---- go/cmd/server/server.go | 23 +++++++++++++++++++---- 2 files changed, 31 insertions(+), 8 deletions(-) diff --git a/go/cmd/server/logging/logging.go b/go/cmd/server/logging/logging.go index bffb118eeb2..760a5cca36d 100644 --- a/go/cmd/server/logging/logging.go +++ b/go/cmd/server/logging/logging.go @@ -148,7 +148,9 @@ func ConvertMemoryBufferToArrowTable(memoryBuffer *MemoryBuffer, fcoSchema *Sche for _, l := range memoryBuffer.logs { // EntityTypes maps an entity name to the specific type and also which index in the entityValues array it is // e.g if an Entity Key is {driver_id, customer_id}, then the driver_id entitytype would be dtype=int64, index=0. - for entityName, idAndType := range fcoSchema.EntityTypes { + // It's in the order of the entities as given by the schema. + for _, entityName := range fcoSchema.Entities { + idAndType := fcoSchema.EntityTypes[entityName] if _, ok := entityNameToEntityValues[entityName]; !ok { entityNameToEntityValues[entityName] = make([]*types.Value, 0) } @@ -170,13 +172,14 @@ func ConvertMemoryBufferToArrowTable(memoryBuffer *MemoryBuffer, fcoSchema *Sche fields := make([]arrow.Field, 0) columns := make([]array.Interface, 0) - for name, val := range entityNameToEntityValues { - valArrowArray, err := gotypes.ProtoValuesToArrowArray(val, arrowMemory, len(columnNameToProtoValueArray)) + for _, entityName := range fcoSchema.Entities { + protoArr := entityNameToEntityValues[entityName] + valArrowArray, err := gotypes.ProtoValuesToArrowArray(protoArr, arrowMemory, len(columnNameToProtoValueArray)) if err != nil { return nil, err } fields = append(fields, arrow.Field{ - Name: name, + Name: entityName, Type: valArrowArray.DataType(), }) columns = append(columns, valArrowArray) @@ -226,6 +229,10 @@ func GetTypesFromFeatureService(featureService *feast.FeatureService, entities [ fvs := make(map[string]*feast.FeatureView) odFvs := make(map[string]*feast.OnDemandFeatureView) + entityNames := make([]string, 0) + for _, entity := range entities { + entityNames = append(entityNames, entity.Joinkey) + } //featureViews, err := fs.listFeatureViews(hideDummyEntity) for _, featureView := range featureViews { @@ -270,6 +277,7 @@ func GetTypesFromFeatureService(featureService *feast.FeatureService, entities [ } } schema := &Schema{ + Entities: entityNames, EntityTypes: entityJoinKeyToType, FeaturesTypes: allFeatureTypes, RequestDataTypes: nil, diff --git a/go/cmd/server/server.go b/go/cmd/server/server.go index 16f50afda61..40002f15efd 100644 --- a/go/cmd/server/server.go +++ b/go/cmd/server/server.go @@ -52,7 +52,8 @@ func (s *servingServiceServer) GetOnlineFeatures(ctx context.Context, request *s FeatureNames: &serving.FeatureList{Val: make([]string, 0)}, }, } - // Entities are currently part of the features as a value + // Entities are currently part of the features as a value and the order that we add it to the resp MetaData + // Need to figure out a way to map the correct entities to the correct ordering entityValues := make(map[string][]*prototypes.Value, 0) for name, values := range request.Entities { resp.Metadata.FeatureNames.Val = append(resp.Metadata.FeatureNames.Val, name) @@ -89,7 +90,7 @@ func (s *servingServiceServer) GetOnlineFeatures(ctx context.Context, request *s return resp, nil } -func generateLogs(s *servingServiceServer, entities map[string][]*prototypes.Value, featureNames []string, features []*serving.GetOnlineFeaturesResponse_FeatureVector, requestData map[string]*prototypes.RepeatedValue) error { +func generateLogs(s *servingServiceServer, entityMap map[string][]*prototypes.Value, featureNames []string, features []*serving.GetOnlineFeaturesResponse_FeatureVector, requestData map[string]*prototypes.RepeatedValue) error { // Add a log with the request context if len(requestData) > 0 { requestContextLog := logging.Log{ @@ -101,7 +102,20 @@ func generateLogs(s *servingServiceServer, entities map[string][]*prototypes.Val if len(features) <= 0 { return nil } + + entityArr, err := s.fs.ListEntities(true) + if err != nil { + return err + } + + joinKeys := make([]string, 0) + for _, entity := range entityArr { + joinKeys = append(joinKeys, entity.Joinkey) + } + numFeatures := len(featureNames) + // Should be equivalent to how many entities there are(each feature row has (entity) number of features) + // acc_rate [] numRows := len(features[0].Values) featureValueLogRows := make([][]*prototypes.Value, numRows) featureStatusLogRows := make([][]serving.FieldStatus, numRows) @@ -117,8 +131,9 @@ func generateLogs(s *servingServiceServer, entities map[string][]*prototypes.Val eventTimestampLogRows[row_idx][idx-1] = features[idx].EventTimestamps[row_idx] } entityRow := make([]*prototypes.Value, 0) - for _, val := range entities { - entityRow = append(entityRow, val[row_idx]) + // ensure that the entity values are in the order that the schema defines which is the order that ListEntities returns the entities + for _, joinKey := range joinKeys { + entityRow = append(entityRow, entityMap[joinKey][row_idx]) } newLog := logging.Log{ EntityValue: entityRow, From 00228f784b54327d7499736b04991c5b23be3998 Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Fri, 8 Apr 2022 16:52:18 -0700 Subject: [PATCH 52/97] Clean up Signed-off-by: Kevin Zhang --- go/cmd/server/server_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/go/cmd/server/server_test.go b/go/cmd/server/server_test.go index 9ef3cdf6bb3..de71fa4823e 100644 --- a/go/cmd/server/server_test.go +++ b/go/cmd/server/server_test.go @@ -49,7 +49,7 @@ func getClient(ctx context.Context, basePath string, enableLogging bool) (servin server := grpc.NewServer() config, err := registry.NewRepoConfigFromFile(getRepoPath(basePath)) - //TODO(kevjumba): either add this officially or talk in design review about what the correct solution for what do with path. + // TODO(kevjumba): either add this officially or talk in design review about what the correct solution for what do with path. // Currently in python we use the path in FileSource but it is not specified in configuration unless it is using file_url? if enableLogging { if config.OfflineStore == nil { @@ -197,7 +197,7 @@ func TestGetOnlineFeaturesSqliteWithLogging(t *testing.T) { time.Sleep(1 * time.Second) assert.Nil(t, err) assert.NotNil(t, response) - // // Wait for logger to flush. + // Wait for logger to flush. // Get the featurenames without the entity order featureNames := response.Metadata.FeatureNames.Val[len(request.Entities):] expectedLogValues, _, _ := GetExpectedLogRows(featureNames, response.Results) From b692e81f89fa76de335ccb55c99b8e86a15930c5 Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Fri, 8 Apr 2022 18:01:07 -0700 Subject: [PATCH 53/97] Update schema functionality Signed-off-by: Kevin Zhang --- go/cmd/server/logging/logging.go | 97 +++++++++++++++++--------------- go/types/typeconversion.go | 40 +++++++++++++ 2 files changed, 91 insertions(+), 46 deletions(-) diff --git a/go/cmd/server/logging/logging.go b/go/cmd/server/logging/logging.go index 760a5cca36d..7e0ed336cde 100644 --- a/go/cmd/server/logging/logging.go +++ b/go/cmd/server/logging/logging.go @@ -145,28 +145,30 @@ func ConvertMemoryBufferToArrowTable(memoryBuffer *MemoryBuffer, fcoSchema *Sche columnNameToStatus := make(map[string][]int32) columnNameToTimestamp := make(map[string][]int64) entityNameToEntityValues := make(map[string][]*types.Value) + for _, l := range memoryBuffer.logs { // EntityTypes maps an entity name to the specific type and also which index in the entityValues array it is // e.g if an Entity Key is {driver_id, customer_id}, then the driver_id entitytype would be dtype=int64, index=0. // It's in the order of the entities as given by the schema. - for _, entityName := range fcoSchema.Entities { - idAndType := fcoSchema.EntityTypes[entityName] + for idx, entityName := range fcoSchema.Entities { if _, ok := entityNameToEntityValues[entityName]; !ok { entityNameToEntityValues[entityName] = make([]*types.Value, 0) } - entityNameToEntityValues[entityName] = append(entityNameToEntityValues[entityName], l.EntityValue[idAndType.index]) + entityNameToEntityValues[entityName] = append(entityNameToEntityValues[entityName], l.EntityValue[idx]) } - for featureName, idAndType := range fcoSchema.FeaturesTypes { + // Contains both fv and odfv feature value types => add them in order of how the appear in the featureService + for idx, featureName := range fcoSchema.Features { + // for featureName, idAndType := range fcoSchema.FeaturesTypes { // populate the proto value arrays with values from memory buffer in separate columns one for each feature name if _, ok := columnNameToProtoValueArray[featureName]; !ok { columnNameToProtoValueArray[featureName] = make([]*types.Value, 0) columnNameToStatus[featureName] = make([]int32, 0) columnNameToTimestamp[featureName] = make([]int64, 0) } - columnNameToProtoValueArray[featureName] = append(columnNameToProtoValueArray[featureName], l.FeatureValues[idAndType.index]) - columnNameToStatus[featureName] = append(columnNameToStatus[featureName], int32(l.FeatureStatuses[idAndType.index])) - columnNameToTimestamp[featureName] = append(columnNameToTimestamp[featureName], l.EventTimestamps[idAndType.index].AsTime().UnixNano()/int64(time.Millisecond)) + columnNameToProtoValueArray[featureName] = append(columnNameToProtoValueArray[featureName], l.FeatureValues[idx]) + columnNameToStatus[featureName] = append(columnNameToStatus[featureName], int32(l.FeatureStatuses[idx])) + columnNameToTimestamp[featureName] = append(columnNameToTimestamp[featureName], l.EventTimestamps[idx].AsTime().UnixNano()/int64(time.Millisecond)) } } @@ -174,29 +176,43 @@ func ConvertMemoryBufferToArrowTable(memoryBuffer *MemoryBuffer, fcoSchema *Sche columns := make([]array.Interface, 0) for _, entityName := range fcoSchema.Entities { protoArr := entityNameToEntityValues[entityName] + if len(protoArr) == 0 { + break + } valArrowArray, err := gotypes.ProtoValuesToArrowArray(protoArr, arrowMemory, len(columnNameToProtoValueArray)) if err != nil { return nil, err } + arrowType, err := gotypes.ValueTypeEnumToArrowType(fcoSchema.EntityTypes[entityName]) + if err != nil { + return nil, err + } fields = append(fields, arrow.Field{ Name: entityName, - Type: valArrowArray.DataType(), + Type: arrowType, }) columns = append(columns, valArrowArray) } - for featureName := range fcoSchema.FeaturesTypes { + for _, featureName := range fcoSchema.Features { - proto_arr := columnNameToProtoValueArray[featureName] - - arrowArray, err := gotypes.ProtoValuesToArrowArray(proto_arr, arrowMemory, len(columnNameToProtoValueArray)) + protoArr := columnNameToProtoValueArray[featureName] + if len(protoArr) == 0 { + break + } + arrowArray, err := gotypes.ProtoValuesToArrowArray(protoArr, arrowMemory, len(columnNameToProtoValueArray)) if err != nil { return nil, err } + arrowType, err := gotypes.ValueTypeEnumToArrowType(fcoSchema.FeaturesTypes[featureName]) + + if err != nil { + return nil, err + } fields = append(fields, arrow.Field{ Name: featureName, - Type: arrowArray.DataType(), + Type: arrowType, }) columns = append(columns, arrowArray) } @@ -204,6 +220,7 @@ func ConvertMemoryBufferToArrowTable(memoryBuffer *MemoryBuffer, fcoSchema *Sche fields, nil, ) + result := array.Record(array.NewRecord(schema, columns, int64(len(memoryBuffer.logs)))) // create an arrow table -> write this to parquet. @@ -213,16 +230,10 @@ func ConvertMemoryBufferToArrowTable(memoryBuffer *MemoryBuffer, fcoSchema *Sche } type Schema struct { - Entities []string - EntityTypes map[string]*IndexAndType - FeaturesTypes map[string]*IndexAndType - RequestDataTypes map[string]*IndexAndType -} - -type IndexAndType struct { - dtype types.ValueType_Enum - // index of the fco(entity, feature, request) as it is ordered in logs - index int + Entities []string + Features []string + EntityTypes map[string]types.ValueType_Enum + FeaturesTypes map[string]types.ValueType_Enum } func GetTypesFromFeatureService(featureService *feast.FeatureService, entities []*feast.Entity, featureViews []*feast.FeatureView, onDemandFeatureViews []*feast.OnDemandFeatureView) (*Schema, error) { @@ -243,18 +254,14 @@ func GetTypesFromFeatureService(featureService *feast.FeatureService, entities [ odFvs[onDemandFeatureView.Base.Name] = onDemandFeatureView } - entityJoinKeyToType := make(map[string]*IndexAndType) - entityNameToJoinKeyMap := make(map[string]string) - for idx, entity := range entities { - entityNameToJoinKeyMap[entity.Name] = entity.Joinkey - entityJoinKeyToType[entity.Joinkey] = &IndexAndType{ - dtype: entity.Valuetype, - index: idx, - } + entityJoinKeyToType := make(map[string]types.ValueType_Enum) + for _, entity := range entities { + entityJoinKeyToType[entity.Joinkey] = entity.Valuetype } - allFeatureTypes := make(map[string]*IndexAndType) + + allFeatureTypes := make(map[string]types.ValueType_Enum) //allRequestDataTypes := make(map[string]*types.ValueType_Enum) - featureIndex := 0 + features := make([]string, 0) for _, featureProjection := range featureService.Projections { // Create copies of FeatureView that may contains the same *FeatureView but // each differentiated by a *FeatureViewProjection @@ -262,25 +269,23 @@ func GetTypesFromFeatureService(featureService *feast.FeatureService, entities [ if fv, ok := fvs[featureViewName]; ok { for _, f := range fv.Base.Features { // add feature to map - allFeatureTypes[f.Name] = &IndexAndType{ - dtype: f.Dtype, - index: featureIndex, - } - featureIndex += 1 + features = append(features, f.Name) + allFeatureTypes[f.Name] = f.Dtype + } + } else if odfv, ok := odFvs[featureViewName]; ok { + for name, valueType := range odfv.GetRequestDataSchema() { + features = append(features, name) + allFeatureTypes[name] = valueType } - } else if _, ok := odFvs[featureViewName]; ok { - // append this -> odfv.getRequestDataSchema() all request data schema - // allRequestDataTypes[featureViewName] = odfv. - featureIndex += 1 } else { return nil, fmt.Errorf("no such feature view found in feature service %s", featureViewName) } } schema := &Schema{ - Entities: entityNames, - EntityTypes: entityJoinKeyToType, - FeaturesTypes: allFeatureTypes, - RequestDataTypes: nil, + Entities: entityNames, + Features: features, + EntityTypes: entityJoinKeyToType, + FeaturesTypes: allFeatureTypes, } return schema, nil } diff --git a/go/types/typeconversion.go b/go/types/typeconversion.go index fc223b02c26..c02768c755d 100644 --- a/go/types/typeconversion.go +++ b/go/types/typeconversion.go @@ -49,6 +49,46 @@ func ProtoTypeToArrowType(sample *types.Value) (arrow.DataType, error) { } } +func ValueTypeEnumToArrowType(t types.ValueType_Enum) (arrow.DataType, error) { + switch t { + case types.ValueType_BYTES: + return arrow.BinaryTypes.Binary, nil + case types.ValueType_STRING: + return arrow.BinaryTypes.String, nil + case types.ValueType_INT32: + return arrow.PrimitiveTypes.Int32, nil + case types.ValueType_INT64: + return arrow.PrimitiveTypes.Int64, nil + case types.ValueType_FLOAT: + return arrow.PrimitiveTypes.Float32, nil + case types.ValueType_DOUBLE: + return arrow.PrimitiveTypes.Float64, nil + case types.ValueType_BOOL: + return arrow.FixedWidthTypes.Boolean, nil + case types.ValueType_BOOL_LIST: + return arrow.ListOf(arrow.FixedWidthTypes.Boolean), nil + case types.ValueType_STRING_LIST: + return arrow.ListOf(arrow.BinaryTypes.String), nil + case types.ValueType_BYTES_LIST: + return arrow.ListOf(arrow.BinaryTypes.Binary), nil + case types.ValueType_INT32_LIST: + return arrow.ListOf(arrow.PrimitiveTypes.Int32), nil + case types.ValueType_INT64_LIST: + return arrow.ListOf(arrow.PrimitiveTypes.Int64), nil + case types.ValueType_FLOAT_LIST: + return arrow.ListOf(arrow.PrimitiveTypes.Float32), nil + case types.ValueType_DOUBLE_LIST: + return arrow.ListOf(arrow.PrimitiveTypes.Float64), nil + case types.ValueType_UNIX_TIMESTAMP: + return arrow.FixedWidthTypes.Time64ns, nil + case types.ValueType_UNIX_TIMESTAMP_LIST: + return arrow.ListOf(arrow.FixedWidthTypes.Time64ns), nil + default: + return nil, + fmt.Errorf("unsupported value type enum in enum to arrow type conversion: %s", t) + } +} + func copyProtoValuesToArrowArray(builder array.Builder, values []*types.Value) error { switch fieldBuilder := builder.(type) { case *array.BooleanBuilder: From 49cd91a3fe76e2637e8066b796d649ce176e2ec3 Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Fri, 8 Apr 2022 18:07:43 -0700 Subject: [PATCH 54/97] Remove xitongsys parquet reader Signed-off-by: Kevin Zhang --- go.mod | 26 ---- go/internal/feast/featurestore_test.go | 199 +------------------------ 2 files changed, 4 insertions(+), 221 deletions(-) diff --git a/go.mod b/go.mod index 09f92cdb7d3..7691c8a12f7 100644 --- a/go.mod +++ b/go.mod @@ -3,12 +3,7 @@ module github.com/feast-dev/feast go 1.17 require ( -<<<<<<< HEAD github.com/apache/arrow/go/v8 v8.0.0-20220408212425-58fe60f59289 -======= - github.com/apache/arrow/go/arrow v0.0.0-20211112161151-bc219186db40 - github.com/apache/arrow/go/v8 v8.0.0-20220405223432-9fa34df27eb1 ->>>>>>> 61c3b837 (Working state) github.com/ghodss/yaml v1.0.0 github.com/go-python/gopy v0.4.0 github.com/go-redis/redis/v8 v8.11.4 @@ -17,39 +12,26 @@ require ( github.com/mattn/go-sqlite3 v1.14.12 github.com/spaolacci/murmur3 v1.1.0 github.com/stretchr/testify v1.7.0 -<<<<<<< HEAD google.golang.org/grpc v1.44.0 google.golang.org/protobuf v1.27.1 -======= github.com/xitongsys/parquet-go v1.6.2 github.com/xitongsys/parquet-go-source v0.0.0-20220315005136-aec0fe3e777c google.golang.org/grpc v1.45.0 google.golang.org/protobuf v1.28.0 ->>>>>>> 61c3b837 (Working state) ) require ( github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c // indirect github.com/andybalholm/brotli v1.0.4 // indirect -<<<<<<< HEAD -<<<<<<< HEAD github.com/apache/thrift v0.15.0 // indirect -======= github.com/apache/arrow/go/v8 v8.0.0-20220405223432-9fa34df27eb1 // indirect -======= ->>>>>>> 61c3b837 (Working state) - github.com/apache/thrift v0.16.0 // indirect ->>>>>>> e0ce4335 (Fix go) github.com/cespare/xxhash/v2 v2.1.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/goccy/go-json v0.9.6 // indirect github.com/golang/snappy v0.0.4 // indirect -<<<<<<< HEAD github.com/gonuts/commander v0.1.0 // indirect github.com/gonuts/flag v0.1.0 // indirect -<<<<<<< HEAD -<<<<<<< HEAD github.com/google/flatbuffers v2.0.5+incompatible // indirect github.com/klauspost/asmfmt v1.3.1 // indirect github.com/klauspost/compress v1.15.1 // indirect @@ -64,18 +46,10 @@ require ( golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3 // indirect golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd // indirect golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27 // indirect -======= ->>>>>>> 8a7ccbe5 (semi working state) - github.com/google/flatbuffers v2.0.0+incompatible // indirect -======= - github.com/google/flatbuffers v2.0.5+incompatible // indirect ->>>>>>> e0ce4335 (Fix go) github.com/google/go-cmp v0.5.7 // indirect github.com/klauspost/asmfmt v1.3.1 // indirect -======= github.com/google/flatbuffers v2.0.6+incompatible // indirect github.com/klauspost/asmfmt v1.3.2 // indirect ->>>>>>> 61c3b837 (Working state) github.com/klauspost/compress v1.15.1 // indirect github.com/klauspost/cpuid/v2 v2.0.12 // indirect github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 // indirect diff --git a/go/internal/feast/featurestore_test.go b/go/internal/feast/featurestore_test.go index 33c1ed4660e..da8fccf73ce 100644 --- a/go/internal/feast/featurestore_test.go +++ b/go/internal/feast/featurestore_test.go @@ -2,16 +2,14 @@ package feast import ( "context" -<<<<<<< HEAD + "path/filepath" + "runtime" + "testing" + "github.com/feast-dev/feast/go/internal/feast/onlinestore" "github.com/feast-dev/feast/go/internal/feast/registry" "github.com/feast-dev/feast/go/protos/feast/types" "github.com/stretchr/testify/assert" -======= ->>>>>>> f7354334 (Make a proof of concept) - "path/filepath" - "runtime" - "testing" "github.com/feast-dev/feast/go/protos/feast/types" "github.com/stretchr/testify/assert" @@ -46,17 +44,8 @@ func TestNewFeatureStore(t *testing.T) { } func TestGetOnlineFeaturesRedis(t *testing.T) { -<<<<<<< HEAD -<<<<<<< HEAD - t.Skip("@todo(achals): feature_repo isn't checked in yet") - config := registry.RepoConfig{ -======= //t.Skip("@todo(achals): feature_repo isn't checked in yet") -======= - t.Skip("@todo(achals): feature_repo isn't checked in yet") ->>>>>>> df2b4a9b (clean up) config := RepoConfig{ ->>>>>>> f7354334 (Make a proof of concept) Project: "feature_repo", Registry: getRegistryPath(), Provider: "local", @@ -75,191 +64,11 @@ func TestGetOnlineFeaturesRedis(t *testing.T) { {Val: &types.Value_Int64Val{Int64Val: 1003}}}}, } -<<<<<<< HEAD fs, err := NewFeatureStore(&config, nil) -======= - fs, err := NewFeatureStore(&config) ->>>>>>> df2b4a9b (clean up) assert.Nil(t, err) ctx := context.Background() response, err := fs.GetOnlineFeatures( ctx, featureNames, nil, entities, map[string]*types.RepeatedValue{}, true) assert.Nil(t, err) -<<<<<<< HEAD assert.Len(t, response, 4) // 3 Features + 1 entity = 4 columns (feature vectors) in response -======= - assert.Len(t, response, 4) // 3 features + 1 entity = 4 columns (feature vectors) in response -} - -func TestGroupingFeatureRefs(t *testing.T) { - viewA := &FeatureView{ - Base: &BaseFeatureView{ - Name: "viewA", - Projection: &FeatureViewProjection{ - NameAlias: "aliasViewA", - }, - }, - Entities: map[string]struct{}{"driver": {}, "customer": {}}, - } - viewB := &FeatureView{ - Base: &BaseFeatureView{Name: "viewB"}, - Entities: map[string]struct{}{"driver": {}, "customer": {}}, - } - viewC := &FeatureView{ - Base: &BaseFeatureView{Name: "viewC"}, - Entities: map[string]struct{}{"driver": {}}, - } - viewD := &FeatureView{ - Base: &BaseFeatureView{Name: "viewD"}, - Entities: map[string]struct{}{"customer": {}}, - } - refGroups, _ := groupFeatureRefs( - []*featureViewAndRefs{ - {view: viewA, featureRefs: []string{"featureA", "featureB"}}, - {view: viewB, featureRefs: []string{"featureC", "featureD"}}, - {view: viewC, featureRefs: []string{"featureE"}}, - {view: viewD, featureRefs: []string{"featureF"}}, - }, - map[string]*types.RepeatedValue{ - "driver_id": {Val: []*types.Value{ - {Val: &types.Value_Int32Val{Int32Val: 0}}, - {Val: &types.Value_Int32Val{Int32Val: 0}}, - {Val: &types.Value_Int32Val{Int32Val: 1}}, - {Val: &types.Value_Int32Val{Int32Val: 1}}, - {Val: &types.Value_Int32Val{Int32Val: 1}}, - }}, - "customer_id": {Val: []*types.Value{ - {Val: &types.Value_Int32Val{Int32Val: 1}}, - {Val: &types.Value_Int32Val{Int32Val: 2}}, - {Val: &types.Value_Int32Val{Int32Val: 3}}, - {Val: &types.Value_Int32Val{Int32Val: 3}}, - {Val: &types.Value_Int32Val{Int32Val: 4}}, - }}, - }, - map[string]string{ - "driver": "driver_id", - "customer": "customer_id", - }, - true, - ) - - assert.Len(t, refGroups, 3) - - // Group 1 - assert.Equal(t, []string{"featureA", "featureB", "featureC", "featureD"}, - refGroups["customer_id,driver_id"].featureNames) - assert.Equal(t, []string{"viewA", "viewA", "viewB", "viewB"}, - refGroups["customer_id,driver_id"].featureViewNames) - assert.Equal(t, []string{ - "aliasViewA__featureA", "aliasViewA__featureB", - "viewB__featureC", "viewB__featureD"}, - refGroups["customer_id,driver_id"].aliasedFeatureNames) - for _, group := range [][]int{{0}, {1}, {2, 3}, {4}} { - assert.Contains(t, refGroups["customer_id,driver_id"].indices, group) - } - - // Group2 - assert.Equal(t, []string{"featureE"}, - refGroups["driver_id"].featureNames) - for _, group := range [][]int{{0, 1}, {2, 3, 4}} { - assert.Contains(t, refGroups["driver_id"].indices, group) - } - - // Group3 - assert.Equal(t, []string{"featureF"}, - refGroups["customer_id"].featureNames) - - for _, group := range [][]int{{0}, {1}, {2, 3}, {4}} { - assert.Contains(t, refGroups["customer_id"].indices, group) - } - -} - -func TestGroupingFeatureRefsWithJoinKeyAliases(t *testing.T) { - viewA := &FeatureView{ - Base: &BaseFeatureView{ - Name: "viewA", - Projection: &FeatureViewProjection{ - Name: "viewA", - JoinKeyMap: map[string]string{"location_id": "destination_id"}, - }, - }, - Entities: map[string]struct{}{"location": {}}, - } - viewB := &FeatureView{ - Base: &BaseFeatureView{Name: "viewB"}, - Entities: map[string]struct{}{"location": {}}, - } - - refGroups, _ := groupFeatureRefs( - []*featureViewAndRefs{ - {view: viewA, featureRefs: []string{"featureA", "featureB"}}, - {view: viewB, featureRefs: []string{"featureC", "featureD"}}, - }, - map[string]*types.RepeatedValue{ - "location_id": {Val: []*types.Value{ - {Val: &types.Value_Int32Val{Int32Val: 0}}, - {Val: &types.Value_Int32Val{Int32Val: 0}}, - {Val: &types.Value_Int32Val{Int32Val: 1}}, - {Val: &types.Value_Int32Val{Int32Val: 1}}, - {Val: &types.Value_Int32Val{Int32Val: 1}}, - }}, - "destination_id": {Val: []*types.Value{ - {Val: &types.Value_Int32Val{Int32Val: 1}}, - {Val: &types.Value_Int32Val{Int32Val: 2}}, - {Val: &types.Value_Int32Val{Int32Val: 3}}, - {Val: &types.Value_Int32Val{Int32Val: 3}}, - {Val: &types.Value_Int32Val{Int32Val: 4}}, - }}, - }, - map[string]string{ - "location": "location_id", - }, - true, - ) - - assert.Len(t, refGroups, 2) - - assert.Equal(t, []string{"featureA", "featureB"}, - refGroups["location_id[destination_id]"].featureNames) - for _, group := range [][]int{{0}, {1}, {2, 3}, {4}} { - assert.Contains(t, refGroups["location_id[destination_id]"].indices, group) - } - - assert.Equal(t, []string{"featureC", "featureD"}, - refGroups["location_id"].featureNames) - for _, group := range [][]int{{0, 1}, {2, 3, 4}} { - assert.Contains(t, refGroups["location_id"].indices, group) - } - -} - -func TestGroupingFeatureRefsWithMissingKey(t *testing.T) { - viewA := &FeatureView{ - Base: &BaseFeatureView{ - Name: "viewA", - Projection: &FeatureViewProjection{ - Name: "viewA", - JoinKeyMap: map[string]string{"location_id": "destination_id"}, - }, - }, - Entities: map[string]struct{}{"location": {}}, - } - - _, err := groupFeatureRefs( - []*featureViewAndRefs{ - {view: viewA, featureRefs: []string{"featureA", "featureB"}}, - }, - map[string]*types.RepeatedValue{ - "location_id": {Val: []*types.Value{ - {Val: &types.Value_Int32Val{Int32Val: 0}}, - }}, - }, - map[string]string{ - "location": "location_id", - }, - true, - ) - assert.Errorf(t, err, "key destination_id is missing in provided entity rows") ->>>>>>> 14784f07 (Fix) } From 4890116ad0fb2d4e058da869304face8c8cceb4b Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Mon, 11 Apr 2022 10:17:24 -0700 Subject: [PATCH 55/97] Clean up Signed-off-by: Kevin Zhang --- go/cmd/server/logging/logging.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/go/cmd/server/logging/logging.go b/go/cmd/server/logging/logging.go index 7e0ed336cde..1d1209bf927 100644 --- a/go/cmd/server/logging/logging.go +++ b/go/cmd/server/logging/logging.go @@ -114,7 +114,7 @@ func (s *LoggingService) flushLogsToOfflineStorage(t time.Time) error { if err != nil { return err } - schema, err := GetTypesFromFeatureService(s.memoryBuffer.featureService, entities, featureViews, odfvs) + schema, err := GetSchemaFromFeatureService(s.memoryBuffer.featureService, entities, featureViews, odfvs) if err != nil { return err } @@ -236,7 +236,7 @@ type Schema struct { FeaturesTypes map[string]types.ValueType_Enum } -func GetTypesFromFeatureService(featureService *feast.FeatureService, entities []*feast.Entity, featureViews []*feast.FeatureView, onDemandFeatureViews []*feast.OnDemandFeatureView) (*Schema, error) { +func GetSchemaFromFeatureService(featureService *feast.FeatureService, entities []*feast.Entity, featureViews []*feast.FeatureView, onDemandFeatureViews []*feast.OnDemandFeatureView) (*Schema, error) { fvs := make(map[string]*feast.FeatureView) odFvs := make(map[string]*feast.OnDemandFeatureView) From 71e1e56a9c4c118dc4557d3724f6cd31f79ba6d6 Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Mon, 11 Apr 2022 11:40:28 -0700 Subject: [PATCH 56/97] Fix go mode Signed-off-by: Kevin Zhang --- go.mod | 24 +----------------------- go.sum | 26 +++++--------------------- 2 files changed, 6 insertions(+), 44 deletions(-) diff --git a/go.mod b/go.mod index 7691c8a12f7..7afa7f588b5 100644 --- a/go.mod +++ b/go.mod @@ -12,10 +12,6 @@ require ( github.com/mattn/go-sqlite3 v1.14.12 github.com/spaolacci/murmur3 v1.1.0 github.com/stretchr/testify v1.7.0 - google.golang.org/grpc v1.44.0 - google.golang.org/protobuf v1.27.1 - github.com/xitongsys/parquet-go v1.6.2 - github.com/xitongsys/parquet-go-source v0.0.0-20220315005136-aec0fe3e777c google.golang.org/grpc v1.45.0 google.golang.org/protobuf v1.28.0 ) @@ -24,30 +20,11 @@ require ( github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c // indirect github.com/andybalholm/brotli v1.0.4 // indirect github.com/apache/thrift v0.15.0 // indirect - github.com/apache/arrow/go/v8 v8.0.0-20220405223432-9fa34df27eb1 // indirect github.com/cespare/xxhash/v2 v2.1.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/goccy/go-json v0.9.6 // indirect github.com/golang/snappy v0.0.4 // indirect - github.com/gonuts/commander v0.1.0 // indirect - github.com/gonuts/flag v0.1.0 // indirect - github.com/google/flatbuffers v2.0.5+incompatible // indirect - github.com/klauspost/asmfmt v1.3.1 // indirect - github.com/klauspost/compress v1.15.1 // indirect - github.com/klauspost/cpuid/v2 v2.0.9 // indirect - github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 // indirect - github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 // indirect - github.com/pierrec/lz4/v4 v4.1.12 // indirect - github.com/pkg/errors v0.9.1 // indirect - github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/zeebo/xxh3 v1.0.1 // indirect - golang.org/x/exp v0.0.0-20211216164055-b2b84827b756 // indirect - golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3 // indirect - golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd // indirect - golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27 // indirect - github.com/google/go-cmp v0.5.7 // indirect - github.com/klauspost/asmfmt v1.3.1 // indirect github.com/google/flatbuffers v2.0.6+incompatible // indirect github.com/klauspost/asmfmt v1.3.2 // indirect github.com/klauspost/compress v1.15.1 // indirect @@ -65,6 +42,7 @@ require ( golang.org/x/tools v0.1.10 // indirect golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect google.golang.org/genproto v0.0.0-20220407144326-9054f6ed7bac // indirect + gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c // indirect ) diff --git a/go.sum b/go.sum index eef3b26b787..47bfd2c5913 100644 --- a/go.sum +++ b/go.sum @@ -150,7 +150,6 @@ github.com/gonuts/commander v0.1.0/go.mod h1:qkb5mSlcWodYgo7vs8ulLnXhfinhZsZcm6+ github.com/gonuts/flag v0.1.0/go.mod h1:ZTmTGtrSPejTo/SRNhCqwLTmiAgyBdCkLYhHrAoBdz4= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/flatbuffers v1.11.0/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= github.com/google/flatbuffers v2.0.5+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= github.com/google/flatbuffers v2.0.6+incompatible h1:XHFReMv7nFFusa+CEokzWbzaYocKXI6C7hdU5Kgh9Lw= github.com/google/flatbuffers v2.0.6+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= @@ -216,8 +215,6 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o github.com/klauspost/asmfmt v1.3.1/go.mod h1:AG8TuvYojzulgDAMCnYn50l/5QV3Bs/tp6j0HLHbNSE= github.com/klauspost/asmfmt v1.3.2 h1:4Ri7ox3EwapiOjCki+hw14RyKk201CN4rzyCJRFLpK4= github.com/klauspost/asmfmt v1.3.2/go.mod h1:AG8TuvYojzulgDAMCnYn50l/5QV3Bs/tp6j0HLHbNSE= -github.com/klauspost/compress v1.9.7/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= -github.com/klauspost/compress v1.13.1/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= github.com/klauspost/compress v1.14.2/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/compress v1.15.1 h1:y9FcTHGyrebwfP0ZZqFiaxTaiDnUrGkJkI+f583BL1A= github.com/klauspost/compress v1.15.1/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= @@ -226,8 +223,9 @@ github.com/klauspost/cpuid/v2 v2.0.12 h1:p9dKCg8i4gmOxtv35DvrYoWqYzQrvEVdjQ762Y0 github.com/klauspost/cpuid/v2 v2.0.12/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuObKfj5c0PQa7c= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= -github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= @@ -299,7 +297,6 @@ github.com/phpdave11/gofpdi v1.0.12/go.mod h1:vBmVV0Do6hSBHC8uKUQ71JGW+ZGQq74llk github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc= github.com/pierrec/lz4 v2.0.5+incompatible h1:2xWsjqPFWcplujydGg4WmhC/6fZqK42wMM8aXeqhl0I= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= -github.com/pierrec/lz4/v4 v4.1.8/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pierrec/lz4/v4 v4.1.12/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pierrec/lz4/v4 v4.1.14 h1:+fL8AQEZtz/ijeNnpduH0bROTu0O3NZAlPjQxGn8LwE= github.com/pierrec/lz4/v4 v4.1.14/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= @@ -367,7 +364,6 @@ github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijb github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= @@ -404,8 +400,6 @@ golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -414,12 +408,6 @@ golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190731235908-ec7cb31e5a56/go.mod h1:JhuoJpWY28nO4Vef9tZUw9qufEGTyX1+7lmHxV5q5G4= golang.org/x/exp v0.0.0-20191002040644-a1355ae1e2c3/go.mod h1:NOZ3BPKG0ec/BKJQgnvsSFpcKLM5xXVWnvZS97DWHgE= -golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= -golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= -golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= golang.org/x/exp v0.0.0-20211216164055-b2b84827b756/go.mod h1:b9TAUYHmRtqA6klRHApnXMnj+OyLce4yF5cZCUbk2ps= golang.org/x/exp v0.0.0-20220407100705-7b9b53b0aca4 h1:K3x+yU+fbot38x5bQbU2QqUAVyYLEktdNH2GxZLnM3U= golang.org/x/exp v0.0.0-20220407100705-7b9b53b0aca4/go.mod h1:lgLbSvA5ygNOMpwM/9anMpWVlVJ7Z+cHWq/eFuinpGE= @@ -446,11 +434,9 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.1.1-0.20191209134235-331c550502dd/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= -golang.org/x/mod v0.6.0-dev.0.20211013180041-c96bc1413d57 h1:LQmS1nU0twXLA96Kt7U9qtHJEbBk3z6Q0V4UXjZkpr4= -golang.org/x/mod v0.6.0-dev.0.20211013180041-c96bc1413d57/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= -golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= golang.org/x/mod v0.6.0-dev.0.20211013180041-c96bc1413d57/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= +golang.org/x/mod v0.6.0-dev.0.20211013180041-c96bc1413d57/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3 h1:kQgndtyPBW/JIYERgdxfwMYh3AVStj88WQTlNDi2a+o= golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -475,7 +461,6 @@ golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= -golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220407224826-aac1ed45d8e3 h1:EN5+DfgmRMvRUrMGERW2gQl3Vc+Z7ZMnI/xdEpPSf0c= @@ -559,7 +544,6 @@ golang.org/x/tools v0.0.0-20200117012304-6edc0a871e69/go.mod h1:TB2adYChydJhpapK golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.8-0.20211029000441-d6a9af8af023/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= -golang.org/x/tools v0.1.9 h1:j9KsMiaP1c3B0OTQGth0/k+miLGTgLsAFUCrF2vLcF8= golang.org/x/tools v0.1.9/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= golang.org/x/tools v0.1.10 h1:QjFRCZxdOhBJ/UNgnBZLbNV13DlbnK0quyivTnXJM20= golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E= @@ -586,7 +570,6 @@ google.golang.org/genproto v0.0.0-20190530194941-fb225487d101/go.mod h1:z3L6/3dT google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto v0.0.0-20210630183607-d20f26d13c79/go.mod h1:yiaVoXHpRzHGyxV3o4DktVWY4mSUErTKaeEOq6C3t3U= google.golang.org/genproto v0.0.0-20220126215142-9970aeb2e350/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20220407144326-9054f6ed7bac h1:qSNTkEN+L2mvWcLgJOR+8bdHX9rN/IdU3A1Ghpfb1Rg= google.golang.org/genproto v0.0.0-20220407144326-9054f6ed7bac/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= @@ -623,8 +606,9 @@ google.golang.org/protobuf v1.28.0 h1:w43yiav+6bVFTBQFZX0r7ipe9JQ1QsbMgHwbBziscL google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= From 050db82548e1615192662f855a67c404cb9c1e4a Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Mon, 11 Apr 2022 12:25:05 -0700 Subject: [PATCH 57/97] Fix tests and errors and everything Signed-off-by: Kevin Zhang --- go/cmd/server/logging/filelogstorage.go | 10 +--- go/cmd/server/logging/logging.go | 19 +++---- go/cmd/server/logging/logging_test.go | 52 +++++++++---------- go/cmd/server/logging/offlinelogstorage.go | 4 +- go/cmd/server/main.go | 2 +- go/cmd/server/server.go | 24 +++------ go/cmd/server/server_test.go | 4 +- go/internal/feast/basefeatureview.go | 51 ------------------ go/internal/feast/featurestore.go | 19 ++++--- go/internal/feast/featureviewprojection.go | 49 ----------------- go/internal/feast/model/basefeatureview.go | 9 ++++ go/internal/feast/model/entity.go | 4 +- go/internal/feast/model/featureservice.go | 8 +-- go/internal/feast/model/featureview.go | 2 +- .../feast/model/featureviewprojection.go | 8 +++ 15 files changed, 83 insertions(+), 182 deletions(-) delete mode 100644 go/internal/feast/basefeatureview.go delete mode 100644 go/internal/feast/featureviewprojection.go diff --git a/go/cmd/server/logging/filelogstorage.go b/go/cmd/server/logging/filelogstorage.go index d8906a74ab4..f9b5e7718ed 100644 --- a/go/cmd/server/logging/filelogstorage.go +++ b/go/cmd/server/logging/filelogstorage.go @@ -17,19 +17,11 @@ type FileLogStorage struct { path string } -type ParquetLog struct { - EntityName string `parquet:"name=entityname, type=BYTE_ARRAY"` - EntityValue string `parquet:"name=entityvalue, type=BYTE_ARRAY"` - FeatureNames []string `parquet:"name=featurenames, type=MAP, convertedtype=LIST, valuetype=BYTE_ARRAY, valueconvertedtype=UTF8"` - FeatureValues []string `parquet:"name=featurevalues, type=MAP, convertedtype=LIST, valuetype=BYTE_ARRAY, valueconvertedtype=UTF8"` - FeatureStatuses []bool `parquet:"name=featurestatuses, type=MAP, convertedtype=LIST, valuetype=BOOLEAN"` - EventTimestamps []int64 `parquet:"name=eventtimestamps, type=MAP, convertedtype=LIST, valuetype=INT64, valueconvertedtype=TIMESTAMP_MILLIS"` -} - func NewFileOfflineStore(project string, offlineStoreConfig map[string]interface{}) (*FileLogStorage, error) { store := FileLogStorage{project: project} var abs_path string var err error + // TODO(kevjumba) remove this default catch. if val, ok := offlineStoreConfig["path"]; !ok { abs_path, err = filepath.Abs("log.parquet") } else { diff --git a/go/cmd/server/logging/logging.go b/go/cmd/server/logging/logging.go index 1d1209bf927..30ee4e742e1 100644 --- a/go/cmd/server/logging/logging.go +++ b/go/cmd/server/logging/logging.go @@ -10,6 +10,7 @@ import ( "github.com/apache/arrow/go/v8/arrow/array" "github.com/apache/arrow/go/v8/arrow/memory" "github.com/feast-dev/feast/go/internal/feast" + "github.com/feast-dev/feast/go/internal/feast/model" "github.com/feast-dev/feast/go/protos/feast/serving" "github.com/feast-dev/feast/go/protos/feast/types" gotypes "github.com/feast-dev/feast/go/types" @@ -27,7 +28,7 @@ type Log struct { } type MemoryBuffer struct { - featureService *feast.FeatureService + featureService *model.FeatureService logs []*Log } @@ -40,10 +41,10 @@ type LoggingService struct { func NewLoggingService(fs *feast.FeatureStore, logChannelCapacity int, featureServiceName string, enableLogging bool) (*LoggingService, error) { // start handler processes? - var featureService *feast.FeatureService = nil + var featureService *model.FeatureService = nil var err error if enableLogging { - featureService, err = fs.GetFeatureService(featureServiceName, fs.GetRepoConfig().Project) + featureService, err = fs.GetFeatureService(featureServiceName) if err != nil { return nil, err } @@ -236,13 +237,13 @@ type Schema struct { FeaturesTypes map[string]types.ValueType_Enum } -func GetSchemaFromFeatureService(featureService *feast.FeatureService, entities []*feast.Entity, featureViews []*feast.FeatureView, onDemandFeatureViews []*feast.OnDemandFeatureView) (*Schema, error) { - fvs := make(map[string]*feast.FeatureView) - odFvs := make(map[string]*feast.OnDemandFeatureView) +func GetSchemaFromFeatureService(featureService *model.FeatureService, entities []*model.Entity, featureViews []*model.FeatureView, onDemandFeatureViews []*model.OnDemandFeatureView) (*Schema, error) { + fvs := make(map[string]*model.FeatureView) + odFvs := make(map[string]*model.OnDemandFeatureView) entityNames := make([]string, 0) for _, entity := range entities { - entityNames = append(entityNames, entity.Joinkey) + entityNames = append(entityNames, entity.JoinKey) } //featureViews, err := fs.listFeatureViews(hideDummyEntity) @@ -256,7 +257,7 @@ func GetSchemaFromFeatureService(featureService *feast.FeatureService, entities entityJoinKeyToType := make(map[string]types.ValueType_Enum) for _, entity := range entities { - entityJoinKeyToType[entity.Joinkey] = entity.Valuetype + entityJoinKeyToType[entity.JoinKey] = entity.ValueType } allFeatureTypes := make(map[string]types.ValueType_Enum) @@ -290,7 +291,7 @@ func GetSchemaFromFeatureService(featureService *feast.FeatureService, entities return schema, nil } -func (s *LoggingService) GetFcos() ([]*feast.Entity, []*feast.FeatureView, []*feast.OnDemandFeatureView, error) { +func (s *LoggingService) GetFcos() ([]*model.Entity, []*model.FeatureView, []*model.OnDemandFeatureView, error) { odfvs, err := s.fs.ListOnDemandFeatureViews() if err != nil { return nil, nil, nil, err diff --git a/go/cmd/server/logging/logging_test.go b/go/cmd/server/logging/logging_test.go index 323020ccf3d..55ccb95af69 100644 --- a/go/cmd/server/logging/logging_test.go +++ b/go/cmd/server/logging/logging_test.go @@ -7,7 +7,7 @@ import ( "github.com/apache/arrow/go/v8/arrow" "github.com/apache/arrow/go/v8/arrow/array" - "github.com/feast-dev/feast/go/internal/feast" + "github.com/feast-dev/feast/go/internal/feast/model" "github.com/feast-dev/feast/go/internal/test" "github.com/feast-dev/feast/go/protos/feast/serving" "github.com/feast-dev/feast/go/protos/feast/types" @@ -42,20 +42,16 @@ func TestLoggingChannelTimeout(t *testing.T) { func TestSchemaTypeRetrieval(t *testing.T) { featureService, entities, featureViews, odfvs := InitializeFeatureRepoVariablesForTest() - schema, err := GetTypesFromFeatureService(featureService, entities, featureViews, odfvs) + schema, err := GetSchemaFromFeatureService(featureService, entities, featureViews, odfvs) assert.Nil(t, err) assert.Contains(t, schema.EntityTypes, "driver_id") - assert.True(t, reflect.DeepEqual(schema.EntityTypes["driver_id"], &IndexAndType{ - dtype: types.ValueType_INT64, - index: 0, - })) + assert.True(t, reflect.DeepEqual(schema.EntityTypes["driver_id"], types.ValueType_INT64)) features := []string{"int64", "float32", "int32", "double"} for idx, featureName := range features { assert.Contains(t, schema.FeaturesTypes, featureName) - assert.Equal(t, schema.FeaturesTypes[featureName].index, idx) + assert.Equal(t, schema.FeaturesTypes[featureName], idx) } - } func TestSerializeToArrowTable(t *testing.T) { @@ -104,61 +100,61 @@ func TestSerializeToArrowTable(t *testing.T) { } } -func InitializeFeatureRepoVariablesForTest() (*feast.FeatureService, []*feast.Entity, []*feast.FeatureView, []*feast.OnDemandFeatureView) { - f1 := feast.NewFeature( +func InitializeFeatureRepoVariablesForTest() (*model.FeatureService, []*model.Entity, []*model.FeatureView, []*model.OnDemandFeatureView) { + f1 := model.NewFeature( "int64", types.ValueType_INT64, ) - f2 := feast.NewFeature( + f2 := model.NewFeature( "float32", types.ValueType_FLOAT, ) - projection1 := feast.NewFeatureViewProjection( + projection1 := model.NewFeatureViewProjection( "featureView1", "", - []*feast.Feature{f1, f2}, + []*model.Feature{f1, f2}, map[string]string{}, ) - baseFeatureView1 := feast.CreateBaseFeatureView( + baseFeatureView1 := model.CreateBaseFeatureView( "featureView1", - []*feast.Feature{f1, f2}, + []*model.Feature{f1, f2}, projection1, ) - featureView1 := feast.CreateFeatureView(baseFeatureView1, nil, map[string]struct{}{}) - entity1 := feast.CreateNewEntity("driver_id", types.ValueType_INT64, "driver_id") - f3 := feast.NewFeature( + featureView1 := model.CreateFeatureView(baseFeatureView1, nil, map[string]struct{}{}) + entity1 := model.CreateNewEntity("driver_id", types.ValueType_INT64, "driver_id") + f3 := model.NewFeature( "int32", types.ValueType_INT32, ) - f4 := feast.NewFeature( + f4 := model.NewFeature( "double", types.ValueType_DOUBLE, ) - projection2 := feast.NewFeatureViewProjection( + projection2 := model.NewFeatureViewProjection( "featureView2", "", - []*feast.Feature{f3, f4}, + []*model.Feature{f3, f4}, map[string]string{}, ) - baseFeatureView2 := feast.CreateBaseFeatureView( + baseFeatureView2 := model.CreateBaseFeatureView( "featureView2", - []*feast.Feature{f3, f4}, + []*model.Feature{f3, f4}, projection2, ) - featureView2 := feast.CreateFeatureView(baseFeatureView2, nil, map[string]struct{}{}) - featureService := feast.NewFeatureService( + featureView2 := model.CreateFeatureView(baseFeatureView2, nil, map[string]struct{}{}) + featureService := model.NewFeatureService( "test_service", "test_project", nil, nil, - []*feast.FeatureViewProjection{projection1, projection2}, + []*model.FeatureViewProjection{projection1, projection2}, ) - return featureService, []*feast.Entity{entity1}, []*feast.FeatureView{featureView1, featureView2}, []*feast.OnDemandFeatureView{} + return featureService, []*model.Entity{entity1}, []*model.FeatureView{featureView1, featureView2}, []*model.OnDemandFeatureView{} } func GenerateLogsAndConvertToArrowTable() (array.Table, error) { featureService, entities, featureViews, odfvs := InitializeFeatureRepoVariablesForTest() - schema, err := GetTypesFromFeatureService(featureService, entities, featureViews, odfvs) + schema, err := GetSchemaFromFeatureService(featureService, entities, featureViews, odfvs) if err != nil { return nil, err } diff --git a/go/cmd/server/logging/offlinelogstorage.go b/go/cmd/server/logging/offlinelogstorage.go index 7f2b36cdaa2..f15cbd8b3ff 100644 --- a/go/cmd/server/logging/offlinelogstorage.go +++ b/go/cmd/server/logging/offlinelogstorage.go @@ -4,7 +4,7 @@ import ( "errors" "github.com/apache/arrow/go/v8/arrow/array" - "github.com/feast-dev/feast/go/internal/feast" + "github.com/feast-dev/feast/go/internal/feast/registry" ) type OfflineLogStorage interface { @@ -21,7 +21,7 @@ func getOfflineStoreType(offlineStoreConfig map[string]interface{}) (string, boo } } -func NewOfflineStore(config *feast.RepoConfig) (OfflineLogStorage, error) { +func NewOfflineStore(config *registry.RepoConfig) (OfflineLogStorage, error) { onlineStoreType, _ := getOfflineStoreType(config.OfflineStore) if onlineStoreType == "file" { offlineStore, err := NewFileOfflineStore(config.Project, config.OfflineStore) diff --git a/go/cmd/server/main.go b/go/cmd/server/main.go index 90068e03971..33d56e0a7a2 100644 --- a/go/cmd/server/main.go +++ b/go/cmd/server/main.go @@ -44,7 +44,7 @@ func main() { } log.Println("Initializing feature store...") - fs, err := feast.NewFeatureStore(repoConfig) + fs, err := feast.NewFeatureStore(repoConfig, nil) if err != nil { log.Fatalln(err) } diff --git a/go/cmd/server/server.go b/go/cmd/server/server.go index 40002f15efd..b9f5ad8e3ff 100644 --- a/go/cmd/server/server.go +++ b/go/cmd/server/server.go @@ -55,37 +55,25 @@ func (s *servingServiceServer) GetOnlineFeatures(ctx context.Context, request *s // Entities are currently part of the features as a value and the order that we add it to the resp MetaData // Need to figure out a way to map the correct entities to the correct ordering entityValues := make(map[string][]*prototypes.Value, 0) - for name, values := range request.Entities { - resp.Metadata.FeatureNames.Val = append(resp.Metadata.FeatureNames.Val, name) - vec := &serving.GetOnlineFeaturesResponse_FeatureVector{ - Values: make([]*prototypes.Value, 0), - Statuses: make([]serving.FieldStatus, 0), - EventTimestamps: make([]*timestamp.Timestamp, 0), - } - resp.Results = append(resp.Results, vec) - valueSlice := make([]*prototypes.Value, 0) - for _, v := range values.Val { - valueSlice = append(valueSlice, v) - vec.Values = append(vec.Values, v) - vec.Statuses = append(vec.Statuses, serving.FieldStatus_PRESENT) - vec.EventTimestamps = append(vec.EventTimestamps, ×tamp.Timestamp{}) - } - entityValues[name] = valueSlice - } featureNames := make([]string, len(featureVectors)) for idx, vector := range featureVectors { + resp.Metadata.FeatureNames.Val = append(resp.Metadata.FeatureNames.Val, vector.Name) featureNames[idx] = vector.Name values, err := types.ArrowValuesToProtoValues(vector.Values) if err != nil { return nil, err } + if _, ok := request.Entities[vector.Name]; ok { + entityValues[vector.Name] = values + } resp.Results = append(resp.Results, &serving.GetOnlineFeaturesResponse_FeatureVector{ Values: values, Statuses: vector.Statuses, EventTimestamps: vector.Timestamps, }) } + go generateLogs(s, entityValues, featureNames, resp.Results, request.RequestContext) return resp, nil } @@ -110,7 +98,7 @@ func generateLogs(s *servingServiceServer, entityMap map[string][]*prototypes.Va joinKeys := make([]string, 0) for _, entity := range entityArr { - joinKeys = append(joinKeys, entity.Joinkey) + joinKeys = append(joinKeys, entity.JoinKey) } numFeatures := len(featureNames) diff --git a/go/cmd/server/server_test.go b/go/cmd/server/server_test.go index de71fa4823e..afc82b8e64b 100644 --- a/go/cmd/server/server_test.go +++ b/go/cmd/server/server_test.go @@ -2,8 +2,6 @@ package main import ( "context" - "github.com/feast-dev/feast/go/internal/feast/registry" - "log" "net" "path/filepath" "reflect" @@ -11,6 +9,8 @@ import ( "testing" "time" + "github.com/feast-dev/feast/go/internal/feast/registry" + "github.com/apache/arrow/go/v8/arrow/array" "github.com/apache/arrow/go/v8/arrow/memory" "github.com/apache/arrow/go/v8/parquet/file" diff --git a/go/internal/feast/basefeatureview.go b/go/internal/feast/basefeatureview.go deleted file mode 100644 index 6f9776bb728..00000000000 --- a/go/internal/feast/basefeatureview.go +++ /dev/null @@ -1,51 +0,0 @@ -package feast - -import ( - "fmt" - - "github.com/feast-dev/feast/go/protos/feast/core" -) - -type BaseFeatureView struct { - Name string - Features []*Feature - Projection *FeatureViewProjection -} - -func NewBaseFeatureView(name string, featureProtos []*core.FeatureSpecV2) *BaseFeatureView { - base := &BaseFeatureView{Name: name} - features := make([]*Feature, len(featureProtos)) - for index, featureSpecV2 := range featureProtos { - features[index] = NewFeatureFromProto(featureSpecV2) - } - base.Features = features - base.Projection = NewFeatureViewProjectionFromDefinition(base) - return base -} - -func (fv *BaseFeatureView) withProjection(projection *FeatureViewProjection) (*BaseFeatureView, error) { - if projection.Name != fv.Name { - return nil, fmt.Errorf("the projection for the %s FeatureView cannot be applied because it differs "+ - "in name; the projection is named %s and the name indicates which "+ - "FeatureView the projection is for", fv.Name, projection.Name) - } - features := make(map[string]bool) - for _, feature := range fv.Features { - features[feature.Name] = true - } - for _, feature := range projection.Features { - if _, ok := features[feature.Name]; !ok { - return nil, fmt.Errorf("the projection for %s cannot be applied because it contains %s which the "+ - "FeatureView doesn't have", projection.Name, feature.Name) - } - } - return &BaseFeatureView{Name: fv.Name, Features: fv.Features, Projection: projection}, nil -} - -func CreateBaseFeatureView(name string, features []*Feature, projection *FeatureViewProjection) *BaseFeatureView { - return &BaseFeatureView{ - Name: name, - Features: features, - Projection: projection, - } -} diff --git a/go/internal/feast/featurestore.go b/go/internal/feast/featurestore.go index b1c0a136521..75d3843728e 100644 --- a/go/internal/feast/featurestore.go +++ b/go/internal/feast/featurestore.go @@ -3,10 +3,8 @@ package feast import ( "context" "errors" - "fmt" - "sort" - "strings" + "github.com/apache/arrow/go/v8/arrow/array" "github.com/apache/arrow/go/v8/arrow/memory" "github.com/feast-dev/feast/go/internal/feast/model" "github.com/feast-dev/feast/go/internal/feast/onlineserving" @@ -16,6 +14,7 @@ import ( "github.com/feast-dev/feast/go/protos/feast/serving" "github.com/feast-dev/feast/go/protos/feast/types" prototypes "github.com/feast-dev/feast/go/protos/feast/types" + "google.golang.org/protobuf/types/known/timestamppb" ) type FeatureStore struct { @@ -48,15 +47,15 @@ type FeatureVector struct { } type featureViewAndRefs struct { - view *FeatureView + view *model.FeatureView featureRefs []string } -func (fs *FeatureStore) Registry() *Registry { +func (fs *FeatureStore) Registry() *registry.Registry { return fs.registry } -func (f *featureViewAndRefs) View() *FeatureView { +func (f *featureViewAndRefs) View() *model.FeatureView { return f.view } @@ -64,6 +63,10 @@ func (f *featureViewAndRefs) FeatureRefs() []string { return f.featureRefs } +func (fs *FeatureStore) GetRepoConfig() *registry.RepoConfig { + return fs.config +} + /* We group all features from a single request by entities they attached to. Thus, we will be able to call online retrieval per entity and not per each feature view. @@ -302,6 +305,10 @@ func (fs *FeatureStore) ListEntities(hideDummyEntity bool) ([]*model.Entity, err return entities, nil } +func (fs *FeatureStore) ListOnDemandFeatureViews() ([]*model.OnDemandFeatureView, error) { + return fs.registry.ListOnDemandFeatureViews(fs.config.Project) +} + /* Group feature views that share the same set of join keys. For each group, we store only unique rows and save indices to retrieve those rows for each requested feature diff --git a/go/internal/feast/featureviewprojection.go b/go/internal/feast/featureviewprojection.go deleted file mode 100644 index 52f498b2d00..00000000000 --- a/go/internal/feast/featureviewprojection.go +++ /dev/null @@ -1,49 +0,0 @@ -package feast - -import ( - "github.com/feast-dev/feast/go/protos/feast/core" -) - -type FeatureViewProjection struct { - Name string - NameAlias string - Features []*Feature - JoinKeyMap map[string]string -} - -func (fv *FeatureViewProjection) nameToUse() string { - if len(fv.NameAlias) == 0 { - return fv.Name - } - return fv.NameAlias -} - -func NewFeatureViewProjectionFromProto(proto *core.FeatureViewProjection) *FeatureViewProjection { - featureProjection := &FeatureViewProjection{Name: proto.FeatureViewName, - NameAlias: proto.FeatureViewNameAlias, - JoinKeyMap: proto.JoinKeyMap, - } - - features := make([]*Feature, len(proto.FeatureColumns)) - for index, featureSpecV2 := range proto.FeatureColumns { - features[index] = NewFeatureFromProto(featureSpecV2) - } - featureProjection.Features = features - return featureProjection -} - -func NewFeatureViewProjectionFromDefinition(base *BaseFeatureView) *FeatureViewProjection { - return &FeatureViewProjection{Name: base.Name, - NameAlias: "", - Features: base.Features, - JoinKeyMap: make(map[string]string), - } -} - -func NewFeatureViewProjection(name string, nameAlias string, features []*Feature, joinKeyMap map[string]string) *FeatureViewProjection { - return &FeatureViewProjection{Name: name, - NameAlias: nameAlias, - Features: features, - JoinKeyMap: joinKeyMap, - } -} diff --git a/go/internal/feast/model/basefeatureview.go b/go/internal/feast/model/basefeatureview.go index 46d8663609f..45d942aab54 100644 --- a/go/internal/feast/model/basefeatureview.go +++ b/go/internal/feast/model/basefeatureview.go @@ -2,6 +2,7 @@ package model import ( "fmt" + "github.com/feast-dev/feast/go/protos/feast/core" ) @@ -56,3 +57,11 @@ func (fv *BaseFeatureView) ProjectWithFeatures(featureNames []string) *FeatureVi Features: features, } } + +func CreateBaseFeatureView(name string, features []*Feature, projection *FeatureViewProjection) *BaseFeatureView { + return &BaseFeatureView{ + Name: name, + Features: features, + Projection: projection, + } +} diff --git a/go/internal/feast/model/entity.go b/go/internal/feast/model/entity.go index d1d817756f9..eff775fa642 100644 --- a/go/internal/feast/model/entity.go +++ b/go/internal/feast/model/entity.go @@ -21,7 +21,7 @@ func NewEntityFromProto(proto *core.Entity) *Entity { func CreateNewEntity(name string, valueType types.ValueType_Enum, joinKey string) *Entity { return &Entity{ Name: name, - Valuetype: valueType, - Joinkey: joinKey, + ValueType: valueType, + JoinKey: joinKey, } } diff --git a/go/internal/feast/model/featureservice.go b/go/internal/feast/model/featureservice.go index 4d73a3c992a..c9ebd998608 100644 --- a/go/internal/feast/model/featureservice.go +++ b/go/internal/feast/model/featureservice.go @@ -28,10 +28,10 @@ func NewFeatureServiceFromProto(proto *core.FeatureService) *FeatureService { func NewFeatureService(name string, project string, createdTimestamp *timestamppb.Timestamp, lastUpdatedTimestamp *timestamppb.Timestamp, projections []*FeatureViewProjection) *FeatureService { return &FeatureService{ - name: name, - project: project, - createdTimestamp: createdTimestamp, - lastUpdatedTimestamp: lastUpdatedTimestamp, + Name: name, + Project: project, + CreatedTimestamp: createdTimestamp, + LastUpdatedTimestamp: lastUpdatedTimestamp, Projections: projections, } } diff --git a/go/internal/feast/model/featureview.go b/go/internal/feast/model/featureview.go index 616df48e946..4500f3c1d67 100644 --- a/go/internal/feast/model/featureview.go +++ b/go/internal/feast/model/featureview.go @@ -48,7 +48,7 @@ func (fs *FeatureView) NewFeatureViewFromBase(base *BaseFeatureView) *FeatureVie func CreateFeatureView(base *BaseFeatureView, ttl *durationpb.Duration, entities map[string]struct{}) *FeatureView { return &FeatureView{ Base: base, - ttl: ttl, + Ttl: ttl, Entities: entities, } } diff --git a/go/internal/feast/model/featureviewprojection.go b/go/internal/feast/model/featureviewprojection.go index e80e8844ed2..48cc7be4ca9 100644 --- a/go/internal/feast/model/featureviewprojection.go +++ b/go/internal/feast/model/featureviewprojection.go @@ -39,3 +39,11 @@ func NewFeatureViewProjectionFromDefinition(base *BaseFeatureView) *FeatureViewP JoinKeyMap: make(map[string]string), } } + +func NewFeatureViewProjection(name string, nameAlias string, features []*Feature, joinKeyMap map[string]string) *FeatureViewProjection { + return &FeatureViewProjection{Name: name, + NameAlias: nameAlias, + Features: features, + JoinKeyMap: joinKeyMap, + } +} From dd235ca942d2e6ddf5bfaa461109dcde3aaba4b1 Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Mon, 11 Apr 2022 12:28:24 -0700 Subject: [PATCH 58/97] Fix tests Signed-off-by: Kevin Zhang --- go/cmd/server/logging/logging_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/go/cmd/server/logging/logging_test.go b/go/cmd/server/logging/logging_test.go index 55ccb95af69..e62b102ec36 100644 --- a/go/cmd/server/logging/logging_test.go +++ b/go/cmd/server/logging/logging_test.go @@ -48,9 +48,10 @@ func TestSchemaTypeRetrieval(t *testing.T) { assert.True(t, reflect.DeepEqual(schema.EntityTypes["driver_id"], types.ValueType_INT64)) features := []string{"int64", "float32", "int32", "double"} + types := []types.ValueType_Enum{*types.ValueType_INT64.Enum(), *types.ValueType_FLOAT.Enum(), *types.ValueType_INT32.Enum(), *types.ValueType_DOUBLE.Enum()} for idx, featureName := range features { assert.Contains(t, schema.FeaturesTypes, featureName) - assert.Equal(t, schema.FeaturesTypes[featureName], idx) + assert.Equal(t, schema.FeaturesTypes[featureName], types[idx]) } } From 7c92d930745ae50f1ba042e85a820dd636ceae3c Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Mon, 11 Apr 2022 12:31:30 -0700 Subject: [PATCH 59/97] Fix Signed-off-by: Kevin Zhang --- go/internal/feast/featurestore_test.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/go/internal/feast/featurestore_test.go b/go/internal/feast/featurestore_test.go index da8fccf73ce..afeb105ab61 100644 --- a/go/internal/feast/featurestore_test.go +++ b/go/internal/feast/featurestore_test.go @@ -10,9 +10,6 @@ import ( "github.com/feast-dev/feast/go/internal/feast/registry" "github.com/feast-dev/feast/go/protos/feast/types" "github.com/stretchr/testify/assert" - - "github.com/feast-dev/feast/go/protos/feast/types" - "github.com/stretchr/testify/assert" ) // Return absolute path to the test_repo registry regardless of the working directory @@ -45,7 +42,7 @@ func TestNewFeatureStore(t *testing.T) { func TestGetOnlineFeaturesRedis(t *testing.T) { //t.Skip("@todo(achals): feature_repo isn't checked in yet") - config := RepoConfig{ + config := registry.RepoConfig{ Project: "feature_repo", Registry: getRegistryPath(), Provider: "local", From 4265efddec1b4b8dabec0300269c1b09f3a31293 Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Mon, 11 Apr 2022 12:35:47 -0700 Subject: [PATCH 60/97] Remove unused code Signed-off-by: Kevin Zhang --- go/cmd/gologging/logging_service.go | 21 ----------- go/cmd/gologging/main.go | 58 ----------------------------- 2 files changed, 79 deletions(-) delete mode 100644 go/cmd/gologging/logging_service.go delete mode 100644 go/cmd/gologging/main.go diff --git a/go/cmd/gologging/logging_service.go b/go/cmd/gologging/logging_service.go deleted file mode 100644 index 1bfaa02cbaf..00000000000 --- a/go/cmd/gologging/logging_service.go +++ /dev/null @@ -1,21 +0,0 @@ -package logging - -import ( - "context" - - "github.com/feast-dev/feast/go/protos/feast/serving" -) - -type loggingServiceServer struct { - channel *chan string - serving.UnimplementedServingServiceServer -} - -func newLoggingServiceServer(channel *chan string) *loggingServiceServer { - return &loggingServiceServer{channel: channel} -} - -func (s *loggingServiceServer) sendLogData(ctx context.Context, log *serving.Log) (*serving.LoggingResponse, error) { - // Actually use the channel and update the channel with the new log. - // return the status -} diff --git a/go/cmd/gologging/main.go b/go/cmd/gologging/main.go deleted file mode 100644 index 2c0cc0884c9..00000000000 --- a/go/cmd/gologging/main.go +++ /dev/null @@ -1,58 +0,0 @@ -package logging - -import ( - "log" - "net" - - "github.com/feast-dev/feast/go/internal/feast" - "github.com/feast-dev/feast/go/protos/feast/serving" - "google.golang.org/grpc" -) - -const ( - flagFeastRepoPath = "FEAST_REPO_PATH" - flagFeastRepoConfig = "FEAST_REPO_CONFIG" - feastServerVersion = "0.18.0" -) - -type FeastEnvConfig struct { - RepoPath string `envconfig:"FEAST_REPO_PATH"` - RepoConfig string `envconfig:"FEAST_REPO_CONFIG"` - SockFile string `envconfig:"FEAST_GRPC_SOCK_FILE"` -} - -type servingServiceServer struct { - fs *feast.FeatureStore - serving.UnimplementedServingServiceServer -} - -func main() { - log_buffer := make(chan string, 1000) - startGrpcServer(log_buffer, "9000") -} - -func startGrpcServer(channel chan string, port string) { - server := newLoggingServiceServer(&channel) - // make new log service - log.Printf("Starting a gRPC server listening on %s\n", port) - lis, err := net.Listen("tcp", port) - if err != nil { - log.Fatalln(err) - } - grpcServer := grpc.NewServer() - defer grpcServer.Stop() - serving.RegisterLoggingServiceServer(grpcServer, server) - // register the servicing service with the grpc server - err = grpcServer.Serve(lis) - if err != nil { - log.Fatalln(err) - } -} - -func processLogs() { - // start a periodic flush - for { - // select function that either receives from channel or starts a subprocess to flush - // probably need to wait for that - } -} From 79cbe42d2698519191baa60ecf30e6cc389ebff3 Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Mon, 11 Apr 2022 12:54:48 -0700 Subject: [PATCH 61/97] Fix Signed-off-by: Kevin Zhang --- go/cmd/server/logging/logging.go | 14 +++---- go/cmd/server/logging/logging_test.go | 2 + go/cmd/server/server.go | 6 +-- go/cmd/server/server_test.go | 11 ++++-- go/internal/feast/featurestore.go | 55 ++------------------------ go/internal/feast/featurestore_test.go | 2 +- go/internal/feast/registry/registry.go | 4 +- 7 files changed, 25 insertions(+), 69 deletions(-) diff --git a/go/cmd/server/logging/logging.go b/go/cmd/server/logging/logging.go index 30ee4e742e1..8d02607499d 100644 --- a/go/cmd/server/logging/logging.go +++ b/go/cmd/server/logging/logging.go @@ -40,7 +40,6 @@ type LoggingService struct { } func NewLoggingService(fs *feast.FeatureStore, logChannelCapacity int, featureServiceName string, enableLogging bool) (*LoggingService, error) { - // start handler processes? var featureService *model.FeatureService = nil var err error if enableLogging { @@ -94,16 +93,20 @@ func (s *LoggingService) processLogs() { } } +// Select that eitheringests new logs that are added to the logging channel, one at a time to add +// to the in memory buffer or flushes all of them synchronously to the OfflineStorage on a time interval. func (s *LoggingService) ProcessMemoryBuffer(t *time.Ticker) { select { case t := <-t.C: s.flushLogsToOfflineStorage(t) case new_log := <-s.logChannel: - log.Printf("Pushing %s to memory.\n", new_log.FeatureValues) + log.Printf("Adding %s to memory.\n", new_log.FeatureValues) s.memoryBuffer.logs = append(s.memoryBuffer.logs, new_log) } } +// Acquires the logging schema from the feature service, converts the memory buffer array of rows of logs and flushes +// them to the offline storage. func (s *LoggingService) flushLogsToOfflineStorage(t time.Time) error { log.Printf("Flushing buffer to offline storage with channel length: %d\n at time: "+t.String(), len(s.memoryBuffer.logs)) offlineStoreType, ok := getOfflineStoreType(s.fs.GetRepoConfig().OfflineStore) @@ -158,9 +161,8 @@ func ConvertMemoryBufferToArrowTable(memoryBuffer *MemoryBuffer, fcoSchema *Sche entityNameToEntityValues[entityName] = append(entityNameToEntityValues[entityName], l.EntityValue[idx]) } - // Contains both fv and odfv feature value types => add them in order of how the appear in the featureService + // Contains both fv and odfv feature value types => they are processed in order of how the appear in the featureService for idx, featureName := range fcoSchema.Features { - // for featureName, idAndType := range fcoSchema.FeaturesTypes { // populate the proto value arrays with values from memory buffer in separate columns one for each feature name if _, ok := columnNameToProtoValueArray[featureName]; !ok { columnNameToProtoValueArray[featureName] = make([]*types.Value, 0) @@ -223,10 +225,8 @@ func ConvertMemoryBufferToArrowTable(memoryBuffer *MemoryBuffer, fcoSchema *Sche ) result := array.Record(array.NewRecord(schema, columns, int64(len(memoryBuffer.logs)))) - // create an arrow table -> write this to parquet. tbl := array.NewTableFromRecords(schema, []array.Record{result}) - // arrow table -> write this to parquet return array.Table(tbl), nil } @@ -245,7 +245,6 @@ func GetSchemaFromFeatureService(featureService *model.FeatureService, entities for _, entity := range entities { entityNames = append(entityNames, entity.JoinKey) } - //featureViews, err := fs.listFeatureViews(hideDummyEntity) for _, featureView := range featureViews { fvs[featureView.Base.Name] = featureView @@ -261,7 +260,6 @@ func GetSchemaFromFeatureService(featureService *model.FeatureService, entities } allFeatureTypes := make(map[string]types.ValueType_Enum) - //allRequestDataTypes := make(map[string]*types.ValueType_Enum) features := make([]string, 0) for _, featureProjection := range featureService.Projections { // Create copies of FeatureView that may contains the same *FeatureView but diff --git a/go/cmd/server/logging/logging_test.go b/go/cmd/server/logging/logging_test.go index e62b102ec36..6364f36915f 100644 --- a/go/cmd/server/logging/logging_test.go +++ b/go/cmd/server/logging/logging_test.go @@ -101,6 +101,7 @@ func TestSerializeToArrowTable(t *testing.T) { } } +// Initialize all dummy featureservice, entities and featureviews/on demand featureviews for testing. func InitializeFeatureRepoVariablesForTest() (*model.FeatureService, []*model.Entity, []*model.FeatureView, []*model.OnDemandFeatureView) { f1 := model.NewFeature( "int64", @@ -153,6 +154,7 @@ func InitializeFeatureRepoVariablesForTest() (*model.FeatureService, []*model.En return featureService, []*model.Entity{entity1}, []*model.FeatureView{featureView1, featureView2}, []*model.OnDemandFeatureView{} } +// Create dummy FeatureService, Entities, and FeatureViews add them to the logger and convert the logs to Arrow table. func GenerateLogsAndConvertToArrowTable() (array.Table, error) { featureService, entities, featureViews, odfvs := InitializeFeatureRepoVariablesForTest() schema, err := GetSchemaFromFeatureService(featureService, entities, featureViews, odfvs) diff --git a/go/cmd/server/server.go b/go/cmd/server/server.go index b9f5ad8e3ff..ccb8a1d2f39 100644 --- a/go/cmd/server/server.go +++ b/go/cmd/server/server.go @@ -54,7 +54,7 @@ func (s *servingServiceServer) GetOnlineFeatures(ctx context.Context, request *s } // Entities are currently part of the features as a value and the order that we add it to the resp MetaData // Need to figure out a way to map the correct entities to the correct ordering - entityValues := make(map[string][]*prototypes.Value, 0) + entityValuesMap := make(map[string][]*prototypes.Value, 0) featureNames := make([]string, len(featureVectors)) for idx, vector := range featureVectors { @@ -65,7 +65,7 @@ func (s *servingServiceServer) GetOnlineFeatures(ctx context.Context, request *s return nil, err } if _, ok := request.Entities[vector.Name]; ok { - entityValues[vector.Name] = values + entityValuesMap[vector.Name] = values } resp.Results = append(resp.Results, &serving.GetOnlineFeaturesResponse_FeatureVector{ Values: values, @@ -74,7 +74,7 @@ func (s *servingServiceServer) GetOnlineFeatures(ctx context.Context, request *s }) } - go generateLogs(s, entityValues, featureNames, resp.Results, request.RequestContext) + go generateLogs(s, entityValuesMap, featureNames, resp.Results, request.RequestContext) return resp, nil } diff --git a/go/cmd/server/server_test.go b/go/cmd/server/server_test.go index afc82b8e64b..90a59b7756a 100644 --- a/go/cmd/server/server_test.go +++ b/go/cmd/server/server_test.go @@ -194,15 +194,18 @@ func TestGetOnlineFeaturesSqliteWithLogging(t *testing.T) { Entities: entities, } response, err := client.GetOnlineFeatures(ctx, request) + // Wait for logger to flush. time.Sleep(1 * time.Second) assert.Nil(t, err) assert.NotNil(t, response) - // Wait for logger to flush. - // Get the featurenames without the entity order + + // Get the featurenames without the entity names that are appended at the front. featureNames := response.Metadata.FeatureNames.Val[len(request.Entities):] + // Generated expected log rows and values + // TODO(kevjumba): implement for timestamp and status expectedLogValues, _, _ := GetExpectedLogRows(featureNames, response.Results) expectedLogValues["driver_id"] = entities["driver_id"] - logPath, err := filepath.Abs(filepath.Join(".", "log.parquet")) + logPath, err := filepath.Abs(filepath.Join(dir, "log.parquet")) assert.Nil(t, err) w, err := logging.CreateOrOpenLogFile(logPath) assert.Nil(t, err) @@ -225,6 +228,7 @@ func TestGetOnlineFeaturesSqliteWithLogging(t *testing.T) { assert.Nil(t, err) assert.Equal(t, len(values), len(expectedLogValues)) + // Need to iterate through and compare because certain values in types.RepeatedValues aren't accurately being compared. for name, val := range values { assert.Equal(t, len(val.Val), len(expectedLogValues[name].Val)) for idx, featureVal := range val.Val { @@ -236,6 +240,7 @@ func TestGetOnlineFeaturesSqliteWithLogging(t *testing.T) { assert.Nil(t, err) } +// Generate the expected log rows based on the resulting feature vector returned from GetOnlineFeatures. func GetExpectedLogRows(featureNames []string, results []*serving.GetOnlineFeaturesResponse_FeatureVector) (map[string]*types.RepeatedValue, [][]int32, [][]int64) { numFeatures := len(featureNames) numRows := len(results[0].Values) diff --git a/go/internal/feast/featurestore.go b/go/internal/feast/featurestore.go index 75d3843728e..b165ca38c5b 100644 --- a/go/internal/feast/featurestore.go +++ b/go/internal/feast/featurestore.go @@ -4,7 +4,6 @@ import ( "context" "errors" - "github.com/apache/arrow/go/v8/arrow/array" "github.com/apache/arrow/go/v8/arrow/memory" "github.com/feast-dev/feast/go/internal/feast/model" "github.com/feast-dev/feast/go/internal/feast/onlineserving" @@ -12,9 +11,7 @@ import ( "github.com/feast-dev/feast/go/internal/feast/registry" "github.com/feast-dev/feast/go/internal/feast/transformation" "github.com/feast-dev/feast/go/protos/feast/serving" - "github.com/feast-dev/feast/go/protos/feast/types" prototypes "github.com/feast-dev/feast/go/protos/feast/types" - "google.golang.org/protobuf/types/known/timestamppb" ) type FeatureStore struct { @@ -32,60 +29,14 @@ type Features struct { FeatureService *model.FeatureService } -/* - FeatureVector type represent result of retrieving single feature for multiple rows. - It can be imagined as a column in output dataframe / table. - It contains of feature name, list of values (across all rows), - list of statuses and list of timestamp. All these lists have equal length. - And this length is also equal to number of entity rows received in request. -*/ -type FeatureVector struct { - Name string - Values array.Interface - Statuses []serving.FieldStatus - Timestamps []*timestamppb.Timestamp -} - -type featureViewAndRefs struct { - view *model.FeatureView - featureRefs []string -} - func (fs *FeatureStore) Registry() *registry.Registry { return fs.registry } -func (f *featureViewAndRefs) View() *model.FeatureView { - return f.view -} - -func (f *featureViewAndRefs) FeatureRefs() []string { - return f.featureRefs -} - func (fs *FeatureStore) GetRepoConfig() *registry.RepoConfig { return fs.config } -/* - We group all features from a single request by entities they attached to. - Thus, we will be able to call online retrieval per entity and not per each feature view. - In this struct we collect all features and views that belongs to a group. - We also store here projected entity keys (only ones that needed to retrieve these features) - and indexes to map result of retrieval into output response. -*/ -type GroupedFeaturesPerEntitySet struct { - // A list of requested feature references of the form featureViewName:featureName that share this entity set - featureNames []string - featureViewNames []string - // full feature references as they supposed to appear in response - aliasedFeatureNames []string - // Entity set as a list of EntityKeys to pass to OnlineRead - entityKeys []*prototypes.EntityKey - // Reversed mapping to project result of retrieval from storage to response - indices [][]int -} - // NewFeatureStore constructs a feature store fat client using the // repo config (contents of feature_store.yaml converted to JSON map). func NewFeatureStore(config *registry.RepoConfig, callback transformation.TransformationCallback) (*FeatureStore, error) { @@ -325,14 +276,14 @@ func (fs *FeatureStore) GetFeatureView(featureViewName string, hideDummyEntity b return fv, nil } -func (fs *FeatureStore) readFromOnlineStore(ctx context.Context, entityRows []*types.EntityKey, +func (fs *FeatureStore) readFromOnlineStore(ctx context.Context, entityRows []*prototypes.EntityKey, requestedFeatureViewNames []string, requestedFeatureNames []string, ) ([][]onlinestore.FeatureData, error) { numRows := len(entityRows) - entityRowsValue := make([]*types.EntityKey, numRows) + entityRowsValue := make([]*prototypes.EntityKey, numRows) for index, entityKey := range entityRows { - entityRowsValue[index] = &types.EntityKey{JoinKeys: entityKey.JoinKeys, EntityValues: entityKey.EntityValues} + entityRowsValue[index] = &prototypes.EntityKey{JoinKeys: entityKey.JoinKeys, EntityValues: entityKey.EntityValues} } return fs.onlineStore.OnlineRead(ctx, entityRowsValue, requestedFeatureViewNames, requestedFeatureNames) } diff --git a/go/internal/feast/featurestore_test.go b/go/internal/feast/featurestore_test.go index afeb105ab61..c8f9049c4a5 100644 --- a/go/internal/feast/featurestore_test.go +++ b/go/internal/feast/featurestore_test.go @@ -41,7 +41,7 @@ func TestNewFeatureStore(t *testing.T) { } func TestGetOnlineFeaturesRedis(t *testing.T) { - //t.Skip("@todo(achals): feature_repo isn't checked in yet") + t.Skip("@todo(achals): feature_repo isn't checked in yet") config := registry.RepoConfig{ Project: "feature_repo", Registry: getRegistryPath(), diff --git a/go/internal/feast/registry/registry.go b/go/internal/feast/registry/registry.go index d3844c307a1..833efee5911 100644 --- a/go/internal/feast/registry/registry.go +++ b/go/internal/feast/registry/registry.go @@ -106,11 +106,11 @@ func (r *Registry) getRegistryProto() (*core.Registry, error) { if err != nil { return registryProto, err } - r.Load(registryProto) + r.load(registryProto) return registryProto, nil } -func (r *Registry) Load(registry *core.Registry) { +func (r *Registry) load(registry *core.Registry) { r.mu.Lock() defer r.mu.Unlock() r.cachedRegistry = registry From 21e99bd951d907de28e77776b146db4de294da96 Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Mon, 11 Apr 2022 15:21:53 -0700 Subject: [PATCH 62/97] Last working commit Signed-off-by: Kevin Zhang --- go/cmd/server/server_test.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/go/cmd/server/server_test.go b/go/cmd/server/server_test.go index 90a59b7756a..7229cdc09ad 100644 --- a/go/cmd/server/server_test.go +++ b/go/cmd/server/server_test.go @@ -173,9 +173,9 @@ func TestGetOnlineFeaturesSqliteWithLogging(t *testing.T) { ctx := context.Background() // Pregenerated using `feast init`. dir := "." - err := test.SetupFeatureRepo(dir) - assert.Nil(t, err) - defer test.CleanUpRepo(dir) + // err := test.SetupFeatureRepo(dir) + // assert.Nil(t, err) + // defer test.CleanUpRepo(dir) client, closer := getClient(ctx, dir, true) defer closer() entities := make(map[string]*types.RepeatedValue) @@ -203,7 +203,7 @@ func TestGetOnlineFeaturesSqliteWithLogging(t *testing.T) { featureNames := response.Metadata.FeatureNames.Val[len(request.Entities):] // Generated expected log rows and values // TODO(kevjumba): implement for timestamp and status - expectedLogValues, _, _ := GetExpectedLogRows(featureNames, response.Results) + expectedLogValues, _, _ := GetExpectedLogRows(featureNames, response.Results[len(request.Entities):]) expectedLogValues["driver_id"] = entities["driver_id"] logPath, err := filepath.Abs(filepath.Join(dir, "log.parquet")) assert.Nil(t, err) @@ -247,17 +247,17 @@ func GetExpectedLogRows(featureNames []string, results []*serving.GetOnlineFeatu featureValueLogRows := make(map[string]*types.RepeatedValue) featureStatusLogRows := make([][]int32, numRows) eventTimestampLogRows := make([][]int64, numRows) - for idx := 1; idx < len(results); idx++ { + for idx := 0; idx < len(results); idx++ { valArray := make([]*types.Value, 0) for row_idx := 0; row_idx < numRows; row_idx++ { featureStatusLogRows[row_idx] = make([]int32, numFeatures) eventTimestampLogRows[row_idx] = make([]int64, numFeatures) valArray = append(valArray, results[idx].Values[row_idx]) - featureStatusLogRows[row_idx][idx-1] = int32(serving.FieldStatus_PRESENT) - eventTimestampLogRows[row_idx][idx-1] = results[idx].EventTimestamps[row_idx].AsTime().UnixNano() / int64(time.Millisecond) + featureStatusLogRows[row_idx][idx] = int32(serving.FieldStatus_PRESENT) + eventTimestampLogRows[row_idx][idx] = results[idx].EventTimestamps[row_idx].AsTime().UnixNano() / int64(time.Millisecond) } - featureValueLogRows[featureNames[idx-1]] = &types.RepeatedValue{ + featureValueLogRows[featureNames[idx]] = &types.RepeatedValue{ Val: valArray, } } From 1b173da0446e21816d685b9ff7bd37c0578961fd Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Mon, 11 Apr 2022 15:23:41 -0700 Subject: [PATCH 63/97] work Signed-off-by: Kevin Zhang --- go/cmd/server/server_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/go/cmd/server/server_test.go b/go/cmd/server/server_test.go index 7229cdc09ad..0c6e9a73ea4 100644 --- a/go/cmd/server/server_test.go +++ b/go/cmd/server/server_test.go @@ -173,9 +173,9 @@ func TestGetOnlineFeaturesSqliteWithLogging(t *testing.T) { ctx := context.Background() // Pregenerated using `feast init`. dir := "." - // err := test.SetupFeatureRepo(dir) - // assert.Nil(t, err) - // defer test.CleanUpRepo(dir) + err := test.SetupFeatureRepo(dir) + assert.Nil(t, err) + defer test.CleanUpRepo(dir) client, closer := getClient(ctx, dir, true) defer closer() entities := make(map[string]*types.RepeatedValue) From b5484c3fb71727df8b926135a15962f18a51520f Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Mon, 11 Apr 2022 15:40:48 -0700 Subject: [PATCH 64/97] Address some changes Signed-off-by: Kevin Zhang --- go/cmd/server/logging/filelogstorage.go | 8 ++-- go/cmd/server/logging/logging.go | 56 ++++++++++++++++++++++ go/cmd/server/server.go | 63 +------------------------ go/cmd/server/server_test.go | 6 +-- 4 files changed, 64 insertions(+), 69 deletions(-) diff --git a/go/cmd/server/logging/filelogstorage.go b/go/cmd/server/logging/filelogstorage.go index f9b5e7718ed..f7ea7a6044c 100644 --- a/go/cmd/server/logging/filelogstorage.go +++ b/go/cmd/server/logging/filelogstorage.go @@ -19,22 +19,22 @@ type FileLogStorage struct { func NewFileOfflineStore(project string, offlineStoreConfig map[string]interface{}) (*FileLogStorage, error) { store := FileLogStorage{project: project} - var abs_path string + var absPath string var err error // TODO(kevjumba) remove this default catch. if val, ok := offlineStoreConfig["path"]; !ok { - abs_path, err = filepath.Abs("log.parquet") + absPath, err = filepath.Abs("log.parquet") } else { result, ok := val.(string) if !ok { return nil, errors.New("cannot convert offlinestore path to string") } - abs_path, err = filepath.Abs(filepath.Join(result, "log.parquet")) + absPath, err = filepath.Abs(filepath.Join(result, "log.parquet")) } if err != nil { return nil, err } - store.path = abs_path + store.path = absPath return &store, nil } diff --git a/go/cmd/server/logging/logging.go b/go/cmd/server/logging/logging.go index 8d02607499d..eb726d54360 100644 --- a/go/cmd/server/logging/logging.go +++ b/go/cmd/server/logging/logging.go @@ -304,3 +304,59 @@ func (s *LoggingService) GetFcos() ([]*model.Entity, []*model.FeatureView, []*mo } return entities, fvs, odfvs, nil } + +func (l *LoggingService) GenerateLogs(entityMap map[string][]*types.Value, featureNames []string, features []*serving.GetOnlineFeaturesResponse_FeatureVector, requestData map[string]*types.RepeatedValue) error { + // Add a log with the request context + if len(requestData) > 0 { + requestContextLog := Log{ + RequestContext: requestData, + } + l.EmitLog(&requestContextLog) + } + + if len(features) <= 0 { + return nil + } + + entityArr, err := l.fs.ListEntities(true) + if err != nil { + return err + } + + joinKeys := make([]string, 0) + for _, entity := range entityArr { + joinKeys = append(joinKeys, entity.JoinKey) + } + + numFeatures := len(featureNames) + // Should be equivalent to how many entities there are(each feature row has (entity) number of features) + // acc_rate [] + numRows := len(features[0].Values) + + for row_idx := 0; row_idx < numRows; row_idx++ { + featureValueLogRow := make([]*types.Value, numFeatures) + featureStatusLogRow := make([]serving.FieldStatus, numFeatures) + eventTimestampLogRow := make([]*timestamppb.Timestamp, numFeatures) + for idx := 0; idx < len(features); idx++ { + featureValueLogRow[idx] = features[idx].Values[row_idx] + featureStatusLogRow[idx] = features[idx].Statuses[row_idx] + eventTimestampLogRow[idx] = features[idx].EventTimestamps[row_idx] + } + entityRow := make([]*types.Value, 0) + // ensure that the entity values are in the order that the schema defines which is the order that ListEntities returns the entities + for _, joinKey := range joinKeys { + entityRow = append(entityRow, entityMap[joinKey][row_idx]) + } + newLog := Log{ + EntityValue: entityRow, + FeatureValues: featureValueLogRow, + FeatureStatuses: featureStatusLogRow, + EventTimestamps: eventTimestampLogRow, + } + err := l.EmitLog(&newLog) + if err != nil { + return err + } + } + return nil +} diff --git a/go/cmd/server/server.go b/go/cmd/server/server.go index ccb8a1d2f39..c5b5996de3b 100644 --- a/go/cmd/server/server.go +++ b/go/cmd/server/server.go @@ -8,7 +8,6 @@ import ( "github.com/feast-dev/feast/go/protos/feast/serving" prototypes "github.com/feast-dev/feast/go/protos/feast/types" "github.com/feast-dev/feast/go/types" - "google.golang.org/protobuf/types/known/timestamppb" ) type servingServiceServer struct { @@ -73,66 +72,6 @@ func (s *servingServiceServer) GetOnlineFeatures(ctx context.Context, request *s EventTimestamps: vector.Timestamps, }) } - - go generateLogs(s, entityValuesMap, featureNames, resp.Results, request.RequestContext) + go s.loggingService.GenerateLogs(entityValuesMap, featureNames, resp.Results[len(request.Entities):], request.RequestContext) return resp, nil } - -func generateLogs(s *servingServiceServer, entityMap map[string][]*prototypes.Value, featureNames []string, features []*serving.GetOnlineFeaturesResponse_FeatureVector, requestData map[string]*prototypes.RepeatedValue) error { - // Add a log with the request context - if len(requestData) > 0 { - requestContextLog := logging.Log{ - RequestContext: requestData, - } - s.loggingService.EmitLog(&requestContextLog) - } - - if len(features) <= 0 { - return nil - } - - entityArr, err := s.fs.ListEntities(true) - if err != nil { - return err - } - - joinKeys := make([]string, 0) - for _, entity := range entityArr { - joinKeys = append(joinKeys, entity.JoinKey) - } - - numFeatures := len(featureNames) - // Should be equivalent to how many entities there are(each feature row has (entity) number of features) - // acc_rate [] - numRows := len(features[0].Values) - featureValueLogRows := make([][]*prototypes.Value, numRows) - featureStatusLogRows := make([][]serving.FieldStatus, numRows) - eventTimestampLogRows := make([][]*timestamppb.Timestamp, numRows) - - for row_idx := 0; row_idx < numRows; row_idx++ { - featureValueLogRows[row_idx] = make([]*prototypes.Value, numFeatures) - featureStatusLogRows[row_idx] = make([]serving.FieldStatus, numFeatures) - eventTimestampLogRows[row_idx] = make([]*timestamppb.Timestamp, numFeatures) - for idx := 1; idx < len(features); idx++ { - featureValueLogRows[row_idx][idx-1] = features[idx].Values[row_idx] - featureStatusLogRows[row_idx][idx-1] = features[idx].Statuses[row_idx] - eventTimestampLogRows[row_idx][idx-1] = features[idx].EventTimestamps[row_idx] - } - entityRow := make([]*prototypes.Value, 0) - // ensure that the entity values are in the order that the schema defines which is the order that ListEntities returns the entities - for _, joinKey := range joinKeys { - entityRow = append(entityRow, entityMap[joinKey][row_idx]) - } - newLog := logging.Log{ - EntityValue: entityRow, - FeatureValues: featureValueLogRows[row_idx], - FeatureStatuses: featureStatusLogRows[row_idx], - EventTimestamps: eventTimestampLogRows[row_idx], - } - err := s.loggingService.EmitLog(&newLog) - if err != nil { - return err - } - } - return nil -} diff --git a/go/cmd/server/server_test.go b/go/cmd/server/server_test.go index 0c6e9a73ea4..7229cdc09ad 100644 --- a/go/cmd/server/server_test.go +++ b/go/cmd/server/server_test.go @@ -173,9 +173,9 @@ func TestGetOnlineFeaturesSqliteWithLogging(t *testing.T) { ctx := context.Background() // Pregenerated using `feast init`. dir := "." - err := test.SetupFeatureRepo(dir) - assert.Nil(t, err) - defer test.CleanUpRepo(dir) + // err := test.SetupFeatureRepo(dir) + // assert.Nil(t, err) + // defer test.CleanUpRepo(dir) client, closer := getClient(ctx, dir, true) defer closer() entities := make(map[string]*types.RepeatedValue) From 545803474e342350b9b6cb9a4aefdfbe1dcf6d17 Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Mon, 11 Apr 2022 15:45:35 -0700 Subject: [PATCH 65/97] More addresses. Signed-off-by: Kevin Zhang --- go/cmd/server/logging/logging.go | 16 +++++++--------- go/cmd/server/logging/logging_test.go | 4 ++-- 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/go/cmd/server/logging/logging.go b/go/cmd/server/logging/logging.go index eb726d54360..6eac0d1c344 100644 --- a/go/cmd/server/logging/logging.go +++ b/go/cmd/server/logging/logging.go @@ -89,13 +89,13 @@ func (s *LoggingService) processLogs() { defer ticker.Stop() for { - s.ProcessMemoryBuffer(ticker) + s.HandleLogFlushing(ticker) } } // Select that eitheringests new logs that are added to the logging channel, one at a time to add // to the in memory buffer or flushes all of them synchronously to the OfflineStorage on a time interval. -func (s *LoggingService) ProcessMemoryBuffer(t *time.Ticker) { +func (s *LoggingService) HandleLogFlushing(t *time.Ticker) { select { case t := <-t.C: s.flushLogsToOfflineStorage(t) @@ -108,7 +108,6 @@ func (s *LoggingService) ProcessMemoryBuffer(t *time.Ticker) { // Acquires the logging schema from the feature service, converts the memory buffer array of rows of logs and flushes // them to the offline storage. func (s *LoggingService) flushLogsToOfflineStorage(t time.Time) error { - log.Printf("Flushing buffer to offline storage with channel length: %d\n at time: "+t.String(), len(s.memoryBuffer.logs)) offlineStoreType, ok := getOfflineStoreType(s.fs.GetRepoConfig().OfflineStore) if !ok { return fmt.Errorf("could not get offline storage type for config: %s", s.fs.GetRepoConfig().OfflineStore) @@ -154,11 +153,11 @@ func ConvertMemoryBufferToArrowTable(memoryBuffer *MemoryBuffer, fcoSchema *Sche // EntityTypes maps an entity name to the specific type and also which index in the entityValues array it is // e.g if an Entity Key is {driver_id, customer_id}, then the driver_id entitytype would be dtype=int64, index=0. // It's in the order of the entities as given by the schema. - for idx, entityName := range fcoSchema.Entities { - if _, ok := entityNameToEntityValues[entityName]; !ok { - entityNameToEntityValues[entityName] = make([]*types.Value, 0) + for idx, joinKey := range fcoSchema.Entities { + if _, ok := entityNameToEntityValues[joinKey]; !ok { + entityNameToEntityValues[joinKey] = make([]*types.Value, 0) } - entityNameToEntityValues[entityName] = append(entityNameToEntityValues[entityName], l.EntityValue[idx]) + entityNameToEntityValues[joinKey] = append(entityNameToEntityValues[joinKey], l.EntityValue[idx]) } // Contains both fv and odfv feature value types => they are processed in order of how the appear in the featureService @@ -266,7 +265,7 @@ func GetSchemaFromFeatureService(featureService *model.FeatureService, entities // each differentiated by a *FeatureViewProjection featureViewName := featureProjection.Name if fv, ok := fvs[featureViewName]; ok { - for _, f := range fv.Base.Features { + for _, f := range fv.Base.Projection.Features { // add feature to map features = append(features, f.Name) allFeatureTypes[f.Name] = f.Dtype @@ -330,7 +329,6 @@ func (l *LoggingService) GenerateLogs(entityMap map[string][]*types.Value, featu numFeatures := len(featureNames) // Should be equivalent to how many entities there are(each feature row has (entity) number of features) - // acc_rate [] numRows := len(features[0].Values) for row_idx := 0; row_idx < numRows; row_idx++ { diff --git a/go/cmd/server/logging/logging_test.go b/go/cmd/server/logging/logging_test.go index 6364f36915f..cc8c30e185e 100644 --- a/go/cmd/server/logging/logging_test.go +++ b/go/cmd/server/logging/logging_test.go @@ -213,8 +213,8 @@ func GenerateLogsAndConvertToArrowTable() (array.Table, error) { loggingService.EmitLog(&log1) loggingService.EmitLog(&log2) - loggingService.ProcessMemoryBuffer(dummyTicker) - loggingService.ProcessMemoryBuffer(dummyTicker) + loggingService.HandleLogFlushing(dummyTicker) + loggingService.HandleLogFlushing(dummyTicker) table, err := ConvertMemoryBufferToArrowTable(loggingService.memoryBuffer, schema) if err != nil { return nil, err From 17b2bf457288f32f770b2313ba8f784170474b86 Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Mon, 11 Apr 2022 16:11:33 -0700 Subject: [PATCH 66/97] Fix more review comments Signed-off-by: Kevin Zhang --- go/cmd/server/logging/filelogstorage_test.go | 41 ++------- go/cmd/server/logging/logging_test.go | 94 +++++++++++++------- go/cmd/server/server_test.go | 11 +-- 3 files changed, 73 insertions(+), 73 deletions(-) diff --git a/go/cmd/server/logging/filelogstorage_test.go b/go/cmd/server/logging/filelogstorage_test.go index b88d1ad9f34..b4926133047 100644 --- a/go/cmd/server/logging/filelogstorage_test.go +++ b/go/cmd/server/logging/filelogstorage_test.go @@ -4,21 +4,18 @@ import ( "path/filepath" "context" - "reflect" "testing" - "github.com/apache/arrow/go/v8/arrow" "github.com/apache/arrow/go/v8/arrow/array" "github.com/apache/arrow/go/v8/arrow/memory" "github.com/apache/arrow/go/v8/parquet/file" "github.com/apache/arrow/go/v8/parquet/pqarrow" "github.com/feast-dev/feast/go/internal/test" - "github.com/feast-dev/feast/go/protos/feast/types" "github.com/stretchr/testify/assert" ) func TestFlushToStorage(t *testing.T) { - table, err := GenerateLogsAndConvertToArrowTable() + table, expectedSchema, expectedColumns, err := GenerateLogsAndConvertToArrowTable() defer table.Release() assert.Nil(t, err) offlineStoreConfig := map[string]interface{}{ @@ -44,44 +41,24 @@ func TestFlushToStorage(t *testing.T) { assert.Nil(t, err) tr := array.NewTableReader(tbl, -1) defer tbl.Release() - expected_schema := map[string]arrow.DataType{ - "driver_id": arrow.PrimitiveTypes.Int64, - "int32": arrow.PrimitiveTypes.Int32, - "double": arrow.PrimitiveTypes.Float64, - "int64": arrow.PrimitiveTypes.Int64, - "float32": arrow.PrimitiveTypes.Float32, - } - - expected_columns := map[string]*types.RepeatedValue{ - "double": { - Val: []*types.Value{{Val: &types.Value_DoubleVal{DoubleVal: 0.97}}, - {Val: &types.Value_DoubleVal{DoubleVal: 8.97}}}}, - "driver_id": { - Val: []*types.Value{{Val: &types.Value_Int64Val{Int64Val: 1001}}, - {Val: &types.Value_Int64Val{Int64Val: 1003}}}}, - "float32": { - Val: []*types.Value{{Val: &types.Value_FloatVal{FloatVal: 0.64}}, - {Val: &types.Value_FloatVal{FloatVal: 1.56}}}}, - "int32": { - Val: []*types.Value{{Val: &types.Value_Int32Val{Int32Val: 55}}, - {Val: &types.Value_Int32Val{Int32Val: 200}}}}, - "int64": { - Val: []*types.Value{{Val: &types.Value_Int64Val{Int64Val: 1000}}, - {Val: &types.Value_Int64Val{Int64Val: 1001}}}}, - } defer tr.Release() for tr.Next() { rec := tr.Record() assert.NotNil(t, rec) for _, field := range rec.Schema().Fields() { - assert.Contains(t, expected_schema, field.Name) - assert.Equal(t, field.Type, expected_schema[field.Name]) + assert.Contains(t, expectedSchema, field.Name) + assert.Equal(t, field.Type, expectedSchema[field.Name]) } values, err := test.GetProtoFromRecord(rec) assert.Nil(t, err) - assert.True(t, reflect.DeepEqual(values, expected_columns)) + for name, val := range values { + assert.Equal(t, len(val.Val), len(expectedColumns[name].Val)) + for idx, featureVal := range val.Val { + assert.Equal(t, featureVal.Val, expectedColumns[name].Val[idx].Val) + } + } } err = test.CleanUpLogs(logPath) diff --git a/go/cmd/server/logging/logging_test.go b/go/cmd/server/logging/logging_test.go index cc8c30e185e..a7ea98d5157 100644 --- a/go/cmd/server/logging/logging_test.go +++ b/go/cmd/server/logging/logging_test.go @@ -11,6 +11,7 @@ import ( "github.com/feast-dev/feast/go/internal/test" "github.com/feast-dev/feast/go/protos/feast/serving" "github.com/feast-dev/feast/go/protos/feast/types" + gotypes "github.com/feast-dev/feast/go/types" "github.com/stretchr/testify/assert" "google.golang.org/protobuf/types/known/timestamppb" ) @@ -56,48 +57,28 @@ func TestSchemaTypeRetrieval(t *testing.T) { } func TestSerializeToArrowTable(t *testing.T) { - table, err := GenerateLogsAndConvertToArrowTable() + table, expectedSchema, expectedColumns, err := GenerateLogsAndConvertToArrowTable() assert.Nil(t, err) defer table.Release() tr := array.NewTableReader(table, -1) - expected_schema := map[string]arrow.DataType{ - "driver_id": arrow.PrimitiveTypes.Int64, - "int32": arrow.PrimitiveTypes.Int32, - "double": arrow.PrimitiveTypes.Float64, - "int64": arrow.PrimitiveTypes.Int64, - "float32": arrow.PrimitiveTypes.Float32, - } - - expected_columns := map[string]*types.RepeatedValue{ - "double": { - Val: []*types.Value{{Val: &types.Value_DoubleVal{DoubleVal: 0.97}}, - {Val: &types.Value_DoubleVal{DoubleVal: 8.97}}}}, - "driver_id": { - Val: []*types.Value{{Val: &types.Value_Int64Val{Int64Val: 1001}}, - {Val: &types.Value_Int64Val{Int64Val: 1003}}}}, - "float32": { - Val: []*types.Value{{Val: &types.Value_FloatVal{FloatVal: 0.64}}, - {Val: &types.Value_FloatVal{FloatVal: 1.56}}}}, - "int32": { - Val: []*types.Value{{Val: &types.Value_Int32Val{Int32Val: 55}}, - {Val: &types.Value_Int32Val{Int32Val: 200}}}}, - "int64": { - Val: []*types.Value{{Val: &types.Value_Int64Val{Int64Val: 1000}}, - {Val: &types.Value_Int64Val{Int64Val: 1001}}}}, - } defer tr.Release() for tr.Next() { rec := tr.Record() assert.NotNil(t, rec) for _, field := range rec.Schema().Fields() { - assert.Contains(t, expected_schema, field.Name) - assert.Equal(t, field.Type, expected_schema[field.Name]) + assert.Contains(t, expectedSchema, field.Name) + assert.Equal(t, field.Type, expectedSchema[field.Name]) } values, err := test.GetProtoFromRecord(rec) assert.Nil(t, err) - assert.True(t, reflect.DeepEqual(values, expected_columns)) + for name, val := range values { + assert.Equal(t, len(val.Val), len(expectedColumns[name].Val)) + for idx, featureVal := range val.Val { + assert.Equal(t, featureVal.Val, expectedColumns[name].Val[idx].Val) + } + } } } @@ -155,15 +136,16 @@ func InitializeFeatureRepoVariablesForTest() (*model.FeatureService, []*model.En } // Create dummy FeatureService, Entities, and FeatureViews add them to the logger and convert the logs to Arrow table. -func GenerateLogsAndConvertToArrowTable() (array.Table, error) { +// Returns arrow table, expected test schema, and expected columns. +func GenerateTestLogsAndConvertToArrowTable() (array.Table, map[string]arrow.DataType, map[string]*types.RepeatedValue, error) { featureService, entities, featureViews, odfvs := InitializeFeatureRepoVariablesForTest() schema, err := GetSchemaFromFeatureService(featureService, entities, featureViews, odfvs) if err != nil { - return nil, err + return nil, nil, nil, err } loggingService, err := NewLoggingService(nil, 2, "", false) if err != nil { - return nil, err + return nil, nil, nil, err } ts := timestamppb.New(time.Now()) log1 := Log{ @@ -207,6 +189,50 @@ func GenerateLogsAndConvertToArrowTable() (array.Table, error) { }, } + expectedSchema := make(map[string]arrow.DataType) + for joinKey, entityType := range schema.EntityTypes { + arrowType, err := gotypes.ValueTypeEnumToArrowType(entityType) + if err != nil { + return nil, nil, nil, err + } + expectedSchema[joinKey] = arrowType + } + for featureName, featureType := range schema.FeaturesTypes { + arrowType, err := gotypes.ValueTypeEnumToArrowType(featureType) + if err != nil { + return nil, nil, nil, err + } + expectedSchema[featureName] = arrowType + } + + expectedColumns := map[string]*types.RepeatedValue{ + "driver_id": { + Val: []*types.Value{ + log1.EntityValue[0], + log2.EntityValue[0]}, + }, + "int64": { + Val: []*types.Value{ + log1.FeatureValues[0], + log2.FeatureValues[0]}, + }, + "float32": { + Val: []*types.Value{ + log1.FeatureValues[1], + log2.FeatureValues[1]}, + }, + "int32": { + Val: []*types.Value{ + log1.FeatureValues[2], + log2.FeatureValues[2]}, + }, + "double": { + Val: []*types.Value{ + log1.FeatureValues[3], + log2.FeatureValues[3]}, + }, + } + dummyTicker := time.NewTicker(10 * time.Second) // stop the ticker so that the logs are not flushed to offline storage dummyTicker.Stop() @@ -217,7 +243,7 @@ func GenerateLogsAndConvertToArrowTable() (array.Table, error) { loggingService.HandleLogFlushing(dummyTicker) table, err := ConvertMemoryBufferToArrowTable(loggingService.memoryBuffer, schema) if err != nil { - return nil, err + return nil, nil, nil, err } - return table, nil + return table, expectedSchema, expectedColumns, nil } diff --git a/go/cmd/server/server_test.go b/go/cmd/server/server_test.go index 7229cdc09ad..bc9978ebc22 100644 --- a/go/cmd/server/server_test.go +++ b/go/cmd/server/server_test.go @@ -173,9 +173,9 @@ func TestGetOnlineFeaturesSqliteWithLogging(t *testing.T) { ctx := context.Background() // Pregenerated using `feast init`. dir := "." - // err := test.SetupFeatureRepo(dir) - // assert.Nil(t, err) - // defer test.CleanUpRepo(dir) + err := test.SetupFeatureRepo(dir) + assert.Nil(t, err) + defer test.CleanUpRepo(dir) client, closer := getClient(ctx, dir, true) defer closer() entities := make(map[string]*types.RepeatedValue) @@ -207,10 +207,7 @@ func TestGetOnlineFeaturesSqliteWithLogging(t *testing.T) { expectedLogValues["driver_id"] = entities["driver_id"] logPath, err := filepath.Abs(filepath.Join(dir, "log.parquet")) assert.Nil(t, err) - w, err := logging.CreateOrOpenLogFile(logPath) - assert.Nil(t, err) - - pf, err := file.NewParquetReader(w) + pf, err := file.OpenParquetFile(logPath, false) assert.Nil(t, err) reader, err := pqarrow.NewFileReader(pf, pqarrow.ArrowReadProperties{}, memory.DefaultAllocator) From 0a1802d0e61e1c28472c085964e679a618250e0d Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Mon, 11 Apr 2022 16:13:41 -0700 Subject: [PATCH 67/97] Fix Signed-off-by: Kevin Zhang --- go/cmd/server/logging/filelogstorage_test.go | 2 +- go/cmd/server/logging/logging_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/go/cmd/server/logging/filelogstorage_test.go b/go/cmd/server/logging/filelogstorage_test.go index b4926133047..4efdb1499a4 100644 --- a/go/cmd/server/logging/filelogstorage_test.go +++ b/go/cmd/server/logging/filelogstorage_test.go @@ -15,7 +15,7 @@ import ( ) func TestFlushToStorage(t *testing.T) { - table, expectedSchema, expectedColumns, err := GenerateLogsAndConvertToArrowTable() + table, expectedSchema, expectedColumns, err := GenerateTestLogsAndConvertToArrowTable() defer table.Release() assert.Nil(t, err) offlineStoreConfig := map[string]interface{}{ diff --git a/go/cmd/server/logging/logging_test.go b/go/cmd/server/logging/logging_test.go index a7ea98d5157..9dae1020d21 100644 --- a/go/cmd/server/logging/logging_test.go +++ b/go/cmd/server/logging/logging_test.go @@ -57,7 +57,7 @@ func TestSchemaTypeRetrieval(t *testing.T) { } func TestSerializeToArrowTable(t *testing.T) { - table, expectedSchema, expectedColumns, err := GenerateLogsAndConvertToArrowTable() + table, expectedSchema, expectedColumns, err := GenerateTestLogsAndConvertToArrowTable() assert.Nil(t, err) defer table.Release() tr := array.NewTableReader(table, -1) From 18c7d60018756686d059c6f1ae5483e9bf750727 Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Mon, 11 Apr 2022 17:36:47 -0700 Subject: [PATCH 68/97] Fix Signed-off-by: Kevin Zhang --- go/cmd/server/logging/filelogstorage.go | 38 ++++++++--- go/cmd/server/logging/filelogstorage_test.go | 11 ++-- go/cmd/server/logging/logging.go | 3 +- go/cmd/server/logging/logging_test.go | 63 +++++++++++-------- go/cmd/server/logging/offlinelogstorage.go | 13 +++- go/cmd/server/server_test.go | 18 +++--- go/internal/feast/model/basefeatureview.go | 8 --- go/internal/feast/model/entity.go | 8 --- go/internal/feast/model/feature.go | 6 -- go/internal/feast/model/featureservice.go | 10 --- .../feast/model/featureviewprojection.go | 8 --- go/internal/test/go_integration_test_utils.go | 42 +++++++++++++ 12 files changed, 138 insertions(+), 90 deletions(-) diff --git a/go/cmd/server/logging/filelogstorage.go b/go/cmd/server/logging/filelogstorage.go index f7ea7a6044c..9a5c78f353d 100644 --- a/go/cmd/server/logging/filelogstorage.go +++ b/go/cmd/server/logging/filelogstorage.go @@ -2,6 +2,7 @@ package logging import ( "errors" + "fmt" "io" "os" "path/filepath" @@ -9,6 +10,7 @@ import ( "github.com/apache/arrow/go/v8/arrow/array" "github.com/apache/arrow/go/v8/parquet" "github.com/apache/arrow/go/v8/parquet/pqarrow" + "github.com/feast-dev/feast/go/internal/feast/registry" ) type FileLogStorage struct { @@ -17,19 +19,39 @@ type FileLogStorage struct { path string } -func NewFileOfflineStore(project string, offlineStoreConfig map[string]interface{}) (*FileLogStorage, error) { +func GetFileConfig(config *registry.RepoConfig) (*OfflineLogStoreConfig, error) { + fileConfig := OfflineLogStoreConfig{ + storeType: "file", + } + if onlineStorePath, ok := config.OfflineStore["path"]; ok { + path, success := onlineStorePath.(string) + if !success { + return &fileConfig, fmt.Errorf("path, %s, cannot be converted to string", path) + } + fileConfig.path = path + } else { + return nil, errors.New("need path for file log storage") + // absPath, err := filepath.Abs(filepath.Join(".", "log.parquet")) + // if err != nil { + // return nil, err + // } + // fileConfig.path = absPath + // return &fileConfig, nil + } + return &fileConfig, nil +} + +// This offline store is currently only used for testing. It will be instantiated during go unit tests to log to file +// and the parquet files will be cleaned up after the test is run. +func NewFileOfflineStore(project string, offlineStoreConfig *OfflineLogStoreConfig) (*FileLogStorage, error) { store := FileLogStorage{project: project} var absPath string var err error // TODO(kevjumba) remove this default catch. - if val, ok := offlineStoreConfig["path"]; !ok { - absPath, err = filepath.Abs("log.parquet") + if offlineStoreConfig.path != "" { + absPath, err = filepath.Abs(offlineStoreConfig.path) } else { - result, ok := val.(string) - if !ok { - return nil, errors.New("cannot convert offlinestore path to string") - } - absPath, err = filepath.Abs(filepath.Join(result, "log.parquet")) + return nil, errors.New("need path for file log storage") } if err != nil { return nil, err diff --git a/go/cmd/server/logging/filelogstorage_test.go b/go/cmd/server/logging/filelogstorage_test.go index 4efdb1499a4..916316990bc 100644 --- a/go/cmd/server/logging/filelogstorage_test.go +++ b/go/cmd/server/logging/filelogstorage_test.go @@ -15,17 +15,18 @@ import ( ) func TestFlushToStorage(t *testing.T) { - table, expectedSchema, expectedColumns, err := GenerateTestLogsAndConvertToArrowTable() + table, expectedSchema, expectedColumns, err := GetTestArrowTableAndExpectedResults() defer table.Release() assert.Nil(t, err) - offlineStoreConfig := map[string]interface{}{ - "path": ".", + offlineStoreConfig := OfflineLogStoreConfig{ + storeType: "file", + path: "./log.parquet", } - fileStore, err := NewFileOfflineStore("test", offlineStoreConfig) + fileStore, err := NewFileOfflineStore("test", &offlineStoreConfig) assert.Nil(t, err) err = fileStore.FlushToStorage(array.Table(table)) assert.Nil(t, err) - logPath, err := filepath.Abs(filepath.Join(".", "log.parquet")) + logPath, err := filepath.Abs(offlineStoreConfig.path) assert.Nil(t, err) w, err := CreateOrOpenLogFile(logPath) assert.Nil(t, err) diff --git a/go/cmd/server/logging/logging.go b/go/cmd/server/logging/logging.go index 6eac0d1c344..3cf5d97f92f 100644 --- a/go/cmd/server/logging/logging.go +++ b/go/cmd/server/logging/logging.go @@ -125,11 +125,10 @@ func (s *LoggingService) flushLogsToOfflineStorage(t time.Time) error { if err != nil { return err } - fileStore, err := NewFileOfflineStore(s.fs.GetRepoConfig().Project, s.fs.GetRepoConfig().OfflineStore) + s.offlineLogStorage.FlushToStorage(table) if err != nil { return err } - fileStore.FlushToStorage(table) s.memoryBuffer.logs = s.memoryBuffer.logs[:0] } else { // Currently don't support any other offline flushing. diff --git a/go/cmd/server/logging/logging_test.go b/go/cmd/server/logging/logging_test.go index 9dae1020d21..a44b39808ab 100644 --- a/go/cmd/server/logging/logging_test.go +++ b/go/cmd/server/logging/logging_test.go @@ -28,7 +28,8 @@ func TestLoggingChannelTimeout(t *testing.T) { } loggingService.EmitLog(&newLog) // Wait for memory buffer flush - time.Sleep(20 * time.Millisecond) + assert.Eventually(t, func() bool { return true }, 100*time.Millisecond, 20*time.Millisecond) + newTs := timestamppb.New(time.Now()) newLog2 := Log{ @@ -57,7 +58,7 @@ func TestSchemaTypeRetrieval(t *testing.T) { } func TestSerializeToArrowTable(t *testing.T) { - table, expectedSchema, expectedColumns, err := GenerateTestLogsAndConvertToArrowTable() + table, expectedSchema, expectedColumns, err := GetTestArrowTableAndExpectedResults() assert.Nil(t, err) defer table.Release() tr := array.NewTableReader(table, -1) @@ -84,48 +85,48 @@ func TestSerializeToArrowTable(t *testing.T) { // Initialize all dummy featureservice, entities and featureviews/on demand featureviews for testing. func InitializeFeatureRepoVariablesForTest() (*model.FeatureService, []*model.Entity, []*model.FeatureView, []*model.OnDemandFeatureView) { - f1 := model.NewFeature( + f1 := test.CreateNewFeature( "int64", types.ValueType_INT64, ) - f2 := model.NewFeature( + f2 := test.CreateNewFeature( "float32", types.ValueType_FLOAT, ) - projection1 := model.NewFeatureViewProjection( + projection1 := test.CreateNewFeatureViewProjection( "featureView1", "", []*model.Feature{f1, f2}, map[string]string{}, ) - baseFeatureView1 := model.CreateBaseFeatureView( + baseFeatureView1 := test.CreateBaseFeatureView( "featureView1", []*model.Feature{f1, f2}, projection1, ) featureView1 := model.CreateFeatureView(baseFeatureView1, nil, map[string]struct{}{}) - entity1 := model.CreateNewEntity("driver_id", types.ValueType_INT64, "driver_id") - f3 := model.NewFeature( + entity1 := test.CreateNewEntity("driver_id", types.ValueType_INT64, "driver_id") + f3 := test.CreateNewFeature( "int32", types.ValueType_INT32, ) - f4 := model.NewFeature( + f4 := test.CreateNewFeature( "double", types.ValueType_DOUBLE, ) - projection2 := model.NewFeatureViewProjection( + projection2 := test.CreateNewFeatureViewProjection( "featureView2", "", []*model.Feature{f3, f4}, map[string]string{}, ) - baseFeatureView2 := model.CreateBaseFeatureView( + baseFeatureView2 := test.CreateBaseFeatureView( "featureView2", []*model.Feature{f3, f4}, projection2, ) featureView2 := model.CreateFeatureView(baseFeatureView2, nil, map[string]struct{}{}) - featureService := model.NewFeatureService( + featureService := test.CreateNewFeatureService( "test_service", "test_project", nil, @@ -137,16 +138,13 @@ func InitializeFeatureRepoVariablesForTest() (*model.FeatureService, []*model.En // Create dummy FeatureService, Entities, and FeatureViews add them to the logger and convert the logs to Arrow table. // Returns arrow table, expected test schema, and expected columns. -func GenerateTestLogsAndConvertToArrowTable() (array.Table, map[string]arrow.DataType, map[string]*types.RepeatedValue, error) { +func GetTestArrowTableAndExpectedResults() (array.Table, map[string]arrow.DataType, map[string]*types.RepeatedValue, error) { featureService, entities, featureViews, odfvs := InitializeFeatureRepoVariablesForTest() schema, err := GetSchemaFromFeatureService(featureService, entities, featureViews, odfvs) if err != nil { return nil, nil, nil, err } - loggingService, err := NewLoggingService(nil, 2, "", false) - if err != nil { - return nil, nil, nil, err - } + ts := timestamppb.New(time.Now()) log1 := Log{ EntityValue: []*types.Value{ @@ -232,18 +230,33 @@ func GenerateTestLogsAndConvertToArrowTable() (array.Table, map[string]arrow.Dat log2.FeatureValues[3]}, }, } + loggingService, err := SetupLoggingServiceWithLogs([]*Log{&log1, &log2}) + if err != nil { + return nil, nil, nil, err + } - dummyTicker := time.NewTicker(10 * time.Second) - // stop the ticker so that the logs are not flushed to offline storage - dummyTicker.Stop() - loggingService.EmitLog(&log1) - - loggingService.EmitLog(&log2) - loggingService.HandleLogFlushing(dummyTicker) - loggingService.HandleLogFlushing(dummyTicker) table, err := ConvertMemoryBufferToArrowTable(loggingService.memoryBuffer, schema) + if err != nil { return nil, nil, nil, err } return table, expectedSchema, expectedColumns, nil } + +func SetupLoggingServiceWithLogs(logs []*Log) (*LoggingService, error) { + loggingService, err := NewLoggingService(nil, len(logs), "", false) + if err != nil { + return nil, err + } + dummyTicker := time.NewTicker(10 * time.Second) + // stop the ticker so that the logs are not flushed to offline storage + dummyTicker.Stop() + for _, log := range logs { + loggingService.EmitLog(log) + } + // manually handle flushing logs + for i := 0; i < len(logs); i++ { + loggingService.HandleLogFlushing(dummyTicker) + } + return loggingService, nil +} diff --git a/go/cmd/server/logging/offlinelogstorage.go b/go/cmd/server/logging/offlinelogstorage.go index f15cbd8b3ff..c3799c13554 100644 --- a/go/cmd/server/logging/offlinelogstorage.go +++ b/go/cmd/server/logging/offlinelogstorage.go @@ -7,7 +7,14 @@ import ( "github.com/feast-dev/feast/go/internal/feast/registry" ) +type OfflineLogStoreConfig struct { + storeType string + project string + path string +} + type OfflineLogStorage interface { + // Todo: Maybe we can add a must implement function that retrieves the correct config based on type FlushToStorage(array.Table) error } @@ -24,7 +31,11 @@ func getOfflineStoreType(offlineStoreConfig map[string]interface{}) (string, boo func NewOfflineStore(config *registry.RepoConfig) (OfflineLogStorage, error) { onlineStoreType, _ := getOfflineStoreType(config.OfflineStore) if onlineStoreType == "file" { - offlineStore, err := NewFileOfflineStore(config.Project, config.OfflineStore) + fileConfig, err := GetFileConfig(config) + if err != nil { + return nil, err + } + offlineStore, err := NewFileOfflineStore(config.Project, fileConfig) return offlineStore, err } else { return nil, errors.New("no offline storage besides file is currently supported") diff --git a/go/cmd/server/server_test.go b/go/cmd/server/server_test.go index bc9978ebc22..16dff614069 100644 --- a/go/cmd/server/server_test.go +++ b/go/cmd/server/server_test.go @@ -53,12 +53,13 @@ func getClient(ctx context.Context, basePath string, enableLogging bool) (servin // Currently in python we use the path in FileSource but it is not specified in configuration unless it is using file_url? if enableLogging { if config.OfflineStore == nil { - config.OfflineStore = map[string]interface{}{ - "path": ".", - } - } else { - config.OfflineStore["path"] = "." + config.OfflineStore = map[string]interface{}{} + } + absPath, err := filepath.Abs(filepath.Join(".", "log.parquet")) + if err != nil { + panic(err) } + config.OfflineStore["path"] = absPath } if err != nil { @@ -68,7 +69,7 @@ func getClient(ctx context.Context, basePath string, enableLogging bool) (servin if err != nil { panic(err) } - generateFeatureService(fs) + CreateFeatureServiceInFeatureStore(fs) loggingService, err := logging.NewLoggingService(fs, 1000, "test_service", enableLogging) if err != nil { panic(err) @@ -166,7 +167,6 @@ func TestGetOnlineFeaturesSqlite(t *testing.T) { assert.True(t, reflect.DeepEqual(response.Results[3].Values, expectedAvgDailyTripsValues)) assert.True(t, reflect.DeepEqual(response.Metadata.FeatureNames.Val, expectedFeatureNamesResp)) - time.Sleep(1 * time.Second) } func TestGetOnlineFeaturesSqliteWithLogging(t *testing.T) { @@ -195,7 +195,7 @@ func TestGetOnlineFeaturesSqliteWithLogging(t *testing.T) { } response, err := client.GetOnlineFeatures(ctx, request) // Wait for logger to flush. - time.Sleep(1 * time.Second) + assert.Eventually(t, func() bool { return true }, 1*time.Second, 200*time.Millisecond) assert.Nil(t, err) assert.NotNil(t, response) @@ -262,7 +262,7 @@ func GetExpectedLogRows(featureNames []string, results []*serving.GetOnlineFeatu return featureValueLogRows, featureStatusLogRows, eventTimestampLogRows } -func generateFeatureService(fs *feast.FeatureStore) { +func CreateFeatureServiceInFeatureStore(fs *feast.FeatureStore) { f1 := core.FeatureSpecV2{ Name: "acc_rate", ValueType: types.ValueType_FLOAT, diff --git a/go/internal/feast/model/basefeatureview.go b/go/internal/feast/model/basefeatureview.go index 45d942aab54..28ef7231fd5 100644 --- a/go/internal/feast/model/basefeatureview.go +++ b/go/internal/feast/model/basefeatureview.go @@ -57,11 +57,3 @@ func (fv *BaseFeatureView) ProjectWithFeatures(featureNames []string) *FeatureVi Features: features, } } - -func CreateBaseFeatureView(name string, features []*Feature, projection *FeatureViewProjection) *BaseFeatureView { - return &BaseFeatureView{ - Name: name, - Features: features, - Projection: projection, - } -} diff --git a/go/internal/feast/model/entity.go b/go/internal/feast/model/entity.go index eff775fa642..ac3a5d5f26e 100644 --- a/go/internal/feast/model/entity.go +++ b/go/internal/feast/model/entity.go @@ -17,11 +17,3 @@ func NewEntityFromProto(proto *core.Entity) *Entity { JoinKey: proto.Spec.JoinKey, } } - -func CreateNewEntity(name string, valueType types.ValueType_Enum, joinKey string) *Entity { - return &Entity{ - Name: name, - ValueType: valueType, - JoinKey: joinKey, - } -} diff --git a/go/internal/feast/model/feature.go b/go/internal/feast/model/feature.go index b5de880fc7f..d833a8901b5 100644 --- a/go/internal/feast/model/feature.go +++ b/go/internal/feast/model/feature.go @@ -15,9 +15,3 @@ func NewFeatureFromProto(proto *core.FeatureSpecV2) *Feature { Dtype: proto.ValueType, } } - -func NewFeature(name string, dtype types.ValueType_Enum) *Feature { - return &Feature{Name: name, - Dtype: dtype, - } -} diff --git a/go/internal/feast/model/featureservice.go b/go/internal/feast/model/featureservice.go index c9ebd998608..5619dd90426 100644 --- a/go/internal/feast/model/featureservice.go +++ b/go/internal/feast/model/featureservice.go @@ -25,13 +25,3 @@ func NewFeatureServiceFromProto(proto *core.FeatureService) *FeatureService { Projections: projections, } } - -func NewFeatureService(name string, project string, createdTimestamp *timestamppb.Timestamp, lastUpdatedTimestamp *timestamppb.Timestamp, projections []*FeatureViewProjection) *FeatureService { - return &FeatureService{ - Name: name, - Project: project, - CreatedTimestamp: createdTimestamp, - LastUpdatedTimestamp: lastUpdatedTimestamp, - Projections: projections, - } -} diff --git a/go/internal/feast/model/featureviewprojection.go b/go/internal/feast/model/featureviewprojection.go index 48cc7be4ca9..e80e8844ed2 100644 --- a/go/internal/feast/model/featureviewprojection.go +++ b/go/internal/feast/model/featureviewprojection.go @@ -39,11 +39,3 @@ func NewFeatureViewProjectionFromDefinition(base *BaseFeatureView) *FeatureViewP JoinKeyMap: make(map[string]string), } } - -func NewFeatureViewProjection(name string, nameAlias string, features []*Feature, joinKeyMap map[string]string) *FeatureViewProjection { - return &FeatureViewProjection{Name: name, - NameAlias: nameAlias, - Features: features, - JoinKeyMap: joinKeyMap, - } -} diff --git a/go/internal/test/go_integration_test_utils.go b/go/internal/test/go_integration_test_utils.go index c95ca034887..9252c85041f 100644 --- a/go/internal/test/go_integration_test_utils.go +++ b/go/internal/test/go_integration_test_utils.go @@ -5,6 +5,7 @@ import ( "fmt" "github.com/apache/arrow/go/v8/arrow/memory" + "google.golang.org/protobuf/types/known/timestamppb" "github.com/apache/arrow/go/v8/arrow" "github.com/apache/arrow/go/v8/parquet/file" @@ -16,6 +17,7 @@ import ( "time" "github.com/apache/arrow/go/v8/arrow/array" + "github.com/feast-dev/feast/go/internal/feast/model" "github.com/feast-dev/feast/go/protos/feast/types" gotypes "github.com/feast-dev/feast/go/types" ) @@ -150,3 +152,43 @@ func GetProtoFromRecord(rec array.Record) (map[string]*types.RepeatedValue, erro func CleanUpLogs(absPath string) error { return os.Remove(absPath) } + +func CreateBaseFeatureView(name string, features []*model.Feature, projection *model.FeatureViewProjection) *model.BaseFeatureView { + return &model.BaseFeatureView{ + Name: name, + Features: features, + Projection: projection, + } +} + +func CreateNewEntity(name string, valueType types.ValueType_Enum, joinKey string) *model.Entity { + return &model.Entity{ + Name: name, + ValueType: valueType, + JoinKey: joinKey, + } +} + +func CreateNewFeature(name string, dtype types.ValueType_Enum) *model.Feature { + return &model.Feature{Name: name, + Dtype: dtype, + } +} + +func CreateNewFeatureService(name string, project string, createdTimestamp *timestamppb.Timestamp, lastUpdatedTimestamp *timestamppb.Timestamp, projections []*model.FeatureViewProjection) *model.FeatureService { + return &model.FeatureService{ + Name: name, + Project: project, + CreatedTimestamp: createdTimestamp, + LastUpdatedTimestamp: lastUpdatedTimestamp, + Projections: projections, + } +} + +func CreateNewFeatureViewProjection(name string, nameAlias string, features []*model.Feature, joinKeyMap map[string]string) *model.FeatureViewProjection { + return &model.FeatureViewProjection{Name: name, + NameAlias: nameAlias, + Features: features, + JoinKeyMap: joinKeyMap, + } +} From a39c9b52b62fc5ae8002a5389dcfe7db2774ef03 Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Mon, 11 Apr 2022 17:39:43 -0700 Subject: [PATCH 69/97] Rename Signed-off-by: Kevin Zhang --- go/cmd/server/logging/logging.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/go/cmd/server/logging/logging.go b/go/cmd/server/logging/logging.go index 3cf5d97f92f..8a788b6b90d 100644 --- a/go/cmd/server/logging/logging.go +++ b/go/cmd/server/logging/logging.go @@ -239,9 +239,9 @@ func GetSchemaFromFeatureService(featureService *model.FeatureService, entities fvs := make(map[string]*model.FeatureView) odFvs := make(map[string]*model.OnDemandFeatureView) - entityNames := make([]string, 0) + joinKeys := make([]string, 0) for _, entity := range entities { - entityNames = append(entityNames, entity.JoinKey) + joinKeys = append(joinKeys, entity.JoinKey) } for _, featureView := range featureViews { @@ -279,7 +279,7 @@ func GetSchemaFromFeatureService(featureService *model.FeatureService, entities } } schema := &Schema{ - Entities: entityNames, + Entities: joinKeys, Features: features, EntityTypes: entityJoinKeyToType, FeaturesTypes: allFeatureTypes, From a4587c326134ec9a532245799cbde1fc652f9561 Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Mon, 11 Apr 2022 19:54:17 -0700 Subject: [PATCH 70/97] Fix Signed-off-by: Kevin Zhang --- go/cmd/server/logging/filelogstorage.go | 16 ++------- go/cmd/server/logging/filelogstorage_test.go | 9 ++--- go/cmd/server/logging/logging.go | 35 ++++++++++++-------- go/cmd/server/logging/logging_test.go | 2 +- go/cmd/server/logging/offlinelogstorage.go | 11 +++--- go/cmd/server/server_test.go | 15 +++++---- 6 files changed, 44 insertions(+), 44 deletions(-) diff --git a/go/cmd/server/logging/filelogstorage.go b/go/cmd/server/logging/filelogstorage.go index 9a5c78f353d..fc9119e4a89 100644 --- a/go/cmd/server/logging/filelogstorage.go +++ b/go/cmd/server/logging/filelogstorage.go @@ -31,12 +31,6 @@ func GetFileConfig(config *registry.RepoConfig) (*OfflineLogStoreConfig, error) fileConfig.path = path } else { return nil, errors.New("need path for file log storage") - // absPath, err := filepath.Abs(filepath.Join(".", "log.parquet")) - // if err != nil { - // return nil, err - // } - // fileConfig.path = absPath - // return &fileConfig, nil } return &fileConfig, nil } @@ -60,7 +54,7 @@ func NewFileOfflineStore(project string, offlineStoreConfig *OfflineLogStoreConf return &store, nil } -func CreateOrOpenLogFile(absPath string) (*os.File, error) { +func OpenLogFile(absPath string) (*os.File, error) { var _, err = os.Stat(absPath) // create file if not exists @@ -71,16 +65,12 @@ func CreateOrOpenLogFile(absPath string) (*os.File, error) { } return file, nil } else { - var file, err = os.OpenFile(absPath, os.O_RDWR, 0644) - if err != nil { - return nil, err - } - return file, nil + return nil, fmt.Errorf("path %s already exists", absPath) } } func (f *FileLogStorage) FlushToStorage(tbl array.Table) error { - w, err := CreateOrOpenLogFile(f.path) + w, err := OpenLogFile(f.path) var writer io.Writer = w if err != nil { return err diff --git a/go/cmd/server/logging/filelogstorage_test.go b/go/cmd/server/logging/filelogstorage_test.go index 916316990bc..44db35e9bc8 100644 --- a/go/cmd/server/logging/filelogstorage_test.go +++ b/go/cmd/server/logging/filelogstorage_test.go @@ -1,9 +1,9 @@ package logging import ( + "context" "path/filepath" - "context" "testing" "github.com/apache/arrow/go/v8/arrow/array" @@ -15,6 +15,7 @@ import ( ) func TestFlushToStorage(t *testing.T) { + ctx := context.Background() table, expectedSchema, expectedColumns, err := GetTestArrowTableAndExpectedResults() defer table.Release() assert.Nil(t, err) @@ -28,11 +29,7 @@ func TestFlushToStorage(t *testing.T) { assert.Nil(t, err) logPath, err := filepath.Abs(offlineStoreConfig.path) assert.Nil(t, err) - w, err := CreateOrOpenLogFile(logPath) - assert.Nil(t, err) - ctx := context.Background() - - pf, err := file.NewParquetReader(w) + pf, err := file.OpenParquetFile(logPath, false) assert.Nil(t, err) reader, err := pqarrow.NewFileReader(pf, pqarrow.ArrowReadProperties{}, memory.DefaultAllocator) diff --git a/go/cmd/server/logging/logging.go b/go/cmd/server/logging/logging.go index 8a788b6b90d..b08a020fdb8 100644 --- a/go/cmd/server/logging/logging.go +++ b/go/cmd/server/logging/logging.go @@ -17,6 +17,9 @@ import ( "google.golang.org/protobuf/types/known/timestamppb" ) +const DEFAULT_LOG_FLUSH_INTERVAL = 100 * time.Millisecond +const DEFAULT_LOG_INSERT_TIMEOUT = 20 * time.Millisecond + type Log struct { // Example: val{int64_val: 5017}, val{int64_val: 1003} EntityValue []*types.Value @@ -37,16 +40,19 @@ type LoggingService struct { logChannel chan *Log fs *feast.FeatureStore offlineLogStorage OfflineLogStorage + logInsertTTl time.Duration + logFlushInterval time.Duration } -func NewLoggingService(fs *feast.FeatureStore, logChannelCapacity int, featureServiceName string, enableLogging bool) (*LoggingService, error) { +func NewLoggingService(fs *feast.FeatureStore, logChannelCapacity int, featureServiceName string, enableLogProcessing bool) (*LoggingService, error) { var featureService *model.FeatureService = nil var err error - if enableLogging { + if fs != nil { featureService, err = fs.GetFeatureService(featureServiceName) if err != nil { return nil, err } + } loggingService := &LoggingService{ @@ -55,20 +61,23 @@ func NewLoggingService(fs *feast.FeatureStore, logChannelCapacity int, featureSe logs: make([]*Log, 0), featureService: featureService, }, - fs: fs, + fs: fs, + logInsertTTl: DEFAULT_LOG_INSERT_TIMEOUT, + logFlushInterval: DEFAULT_LOG_FLUSH_INTERVAL, } - if !enableLogging || fs == nil { - loggingService.offlineLogStorage = nil - } else { + if fs != nil { offlineLogStorage, err := NewOfflineStore(fs.GetRepoConfig()) loggingService.offlineLogStorage = offlineLogStorage - if err != nil { return nil, err } - // Start goroutine to process logs + } + + // Start goroutine to process logs + if enableLogProcessing { go loggingService.processLogs() + } return loggingService, nil } @@ -77,7 +86,7 @@ func (s *LoggingService) EmitLog(l *Log) error { select { case s.logChannel <- l: return nil - case <-time.After(20 * time.Millisecond): + case <-time.After(s.logInsertTTl): return fmt.Errorf("could not add to log channel with capacity %d. Operation timed out. Current log channel length is %d", cap(s.logChannel), len(s.logChannel)) } } @@ -85,7 +94,7 @@ func (s *LoggingService) EmitLog(l *Log) error { func (s *LoggingService) processLogs() { // start a periodic flush // TODO(kevjumba): set param so users can configure flushing duration - ticker := time.NewTicker(100 * time.Millisecond) + ticker := time.NewTicker(s.logFlushInterval) defer ticker.Stop() for { @@ -263,9 +272,8 @@ func GetSchemaFromFeatureService(featureService *model.FeatureService, entities // Create copies of FeatureView that may contains the same *FeatureView but // each differentiated by a *FeatureViewProjection featureViewName := featureProjection.Name - if fv, ok := fvs[featureViewName]; ok { - for _, f := range fv.Base.Projection.Features { - // add feature to map + if _, ok := fvs[featureViewName]; ok { + for _, f := range featureProjection.Features { features = append(features, f.Name) allFeatureTypes[f.Name] = f.Dtype } @@ -278,6 +286,7 @@ func GetSchemaFromFeatureService(featureService *model.FeatureService, entities return nil, fmt.Errorf("no such feature view found in feature service %s", featureViewName) } } + schema := &Schema{ Entities: joinKeys, Features: features, diff --git a/go/cmd/server/logging/logging_test.go b/go/cmd/server/logging/logging_test.go index a44b39808ab..f9ab351108d 100644 --- a/go/cmd/server/logging/logging_test.go +++ b/go/cmd/server/logging/logging_test.go @@ -28,7 +28,7 @@ func TestLoggingChannelTimeout(t *testing.T) { } loggingService.EmitLog(&newLog) // Wait for memory buffer flush - assert.Eventually(t, func() bool { return true }, 100*time.Millisecond, 20*time.Millisecond) + assert.Eventually(t, func() bool { return true }, 100*time.Millisecond, DEFAULT_LOG_INSERT_TIMEOUT) newTs := timestamppb.New(time.Now()) diff --git a/go/cmd/server/logging/offlinelogstorage.go b/go/cmd/server/logging/offlinelogstorage.go index c3799c13554..1a0f4142554 100644 --- a/go/cmd/server/logging/offlinelogstorage.go +++ b/go/cmd/server/logging/offlinelogstorage.go @@ -19,9 +19,9 @@ type OfflineLogStorage interface { } func getOfflineStoreType(offlineStoreConfig map[string]interface{}) (string, bool) { - if onlineStoreType, ok := offlineStoreConfig["type"]; !ok { + if onlineStoreType, ok := offlineStoreConfig["storeType"]; !ok { // Assume file for case of no specified. - return "file", true + return "", true } else { result, ok := onlineStoreType.(string) return result, ok @@ -29,8 +29,11 @@ func getOfflineStoreType(offlineStoreConfig map[string]interface{}) (string, boo } func NewOfflineStore(config *registry.RepoConfig) (OfflineLogStorage, error) { - onlineStoreType, _ := getOfflineStoreType(config.OfflineStore) - if onlineStoreType == "file" { + offlineStoreType, _ := getOfflineStoreType(config.OfflineStore) + if offlineStoreType == "" { + // No offline store specified. + return nil, nil + } else if offlineStoreType == "file" { fileConfig, err := GetFileConfig(config) if err != nil { return nil, err diff --git a/go/cmd/server/server_test.go b/go/cmd/server/server_test.go index 16dff614069..c3a1c37769c 100644 --- a/go/cmd/server/server_test.go +++ b/go/cmd/server/server_test.go @@ -42,7 +42,7 @@ func getRepoPath(basePath string) string { } // Starts a new grpc server, registers the serving service and returns a client. -func getClient(ctx context.Context, basePath string, enableLogging bool) (serving.ServingServiceClient, func()) { +func getClient(ctx context.Context, offlineStoreType string, basePath string, enableLogging bool) (serving.ServingServiceClient, func()) { buffer := 1024 * 1024 listener := bufconn.Listen(buffer) @@ -60,6 +60,7 @@ func getClient(ctx context.Context, basePath string, enableLogging bool) (servin panic(err) } config.OfflineStore["path"] = absPath + config.OfflineStore["storeType"] = offlineStoreType } if err != nil { @@ -104,7 +105,7 @@ func TestGetFeastServingInfo(t *testing.T) { err := test.SetupFeatureRepo(dir) assert.Nil(t, err) defer test.CleanUpRepo(dir) - client, closer := getClient(ctx, dir, false) + client, closer := getClient(ctx, "", dir, false) defer closer() response, err := client.GetFeastServingInfo(ctx, &serving.GetFeastServingInfoRequest{}) assert.Nil(t, err) @@ -118,7 +119,7 @@ func TestGetOnlineFeaturesSqlite(t *testing.T) { err := test.SetupFeatureRepo(dir) assert.Nil(t, err) defer test.CleanUpRepo(dir) - client, closer := getClient(ctx, dir, false) + client, closer := getClient(ctx, "", dir, false) defer closer() entities := make(map[string]*types.RepeatedValue) entities["driver_id"] = &types.RepeatedValue{ @@ -176,7 +177,7 @@ func TestGetOnlineFeaturesSqliteWithLogging(t *testing.T) { err := test.SetupFeatureRepo(dir) assert.Nil(t, err) defer test.CleanUpRepo(dir) - client, closer := getClient(ctx, dir, true) + client, closer := getClient(ctx, "file", dir, true) defer closer() entities := make(map[string]*types.RepeatedValue) entities["driver_id"] = &types.RepeatedValue{ @@ -195,7 +196,7 @@ func TestGetOnlineFeaturesSqliteWithLogging(t *testing.T) { } response, err := client.GetOnlineFeatures(ctx, request) // Wait for logger to flush. - assert.Eventually(t, func() bool { return true }, 1*time.Second, 200*time.Millisecond) + assert.Eventually(t, func() bool { return true }, 1*time.Second, logging.DEFAULT_LOG_FLUSH_INTERVAL) assert.Nil(t, err) assert.NotNil(t, response) @@ -233,8 +234,8 @@ func TestGetOnlineFeaturesSqliteWithLogging(t *testing.T) { } } } - err = test.CleanUpLogs(logPath) - assert.Nil(t, err) + // err = test.CleanUpLogs(logPath) + // assert.Nil(t, err) } // Generate the expected log rows based on the resulting feature vector returned from GetOnlineFeatures. From 83c5f89382d58ef1b24e468f569e3dab1e316223 Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Mon, 11 Apr 2022 20:36:59 -0700 Subject: [PATCH 71/97] Add request id Signed-off-by: Kevin Zhang --- go.mod | 2 +- go.sum | 2 ++ go/cmd/server/logging/logging.go | 13 ++++++++++++- go/cmd/server/server.go | 9 ++++++++- go/cmd/server/server_test.go | 18 ++++++++++++------ 5 files changed, 35 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index 7afa7f588b5..585072e9fb1 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( github.com/go-python/gopy v0.4.0 github.com/go-redis/redis/v8 v8.11.4 github.com/golang/protobuf v1.5.2 - github.com/google/uuid v1.2.0 + github.com/google/uuid v1.3.0 github.com/mattn/go-sqlite3 v1.14.12 github.com/spaolacci/murmur3 v1.1.0 github.com/stretchr/testify v1.7.0 diff --git a/go.sum b/go.sum index 47bfd2c5913..8d9b3bd7c5c 100644 --- a/go.sum +++ b/go.sum @@ -168,6 +168,8 @@ github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+ github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.2.0 h1:qJYtXnJRWmpe7m/3XlyhrsLrEURqHRM2kxzoxXqyUDs= github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= +github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= diff --git a/go/cmd/server/logging/logging.go b/go/cmd/server/logging/logging.go index b08a020fdb8..d522ca34888 100644 --- a/go/cmd/server/logging/logging.go +++ b/go/cmd/server/logging/logging.go @@ -28,6 +28,7 @@ type Log struct { FeatureStatuses []serving.FieldStatus EventTimestamps []*timestamppb.Timestamp RequestContext map[string]*types.RepeatedValue + RequestId string } type MemoryBuffer struct { @@ -157,6 +158,8 @@ func ConvertMemoryBufferToArrowTable(memoryBuffer *MemoryBuffer, fcoSchema *Sche columnNameToTimestamp := make(map[string][]int64) entityNameToEntityValues := make(map[string][]*types.Value) + strBuilder := array.NewStringBuilder(arrowMemory) + for _, l := range memoryBuffer.logs { // EntityTypes maps an entity name to the specific type and also which index in the entityValues array it is // e.g if an Entity Key is {driver_id, customer_id}, then the driver_id entitytype would be dtype=int64, index=0. @@ -180,6 +183,7 @@ func ConvertMemoryBufferToArrowTable(memoryBuffer *MemoryBuffer, fcoSchema *Sche columnNameToStatus[featureName] = append(columnNameToStatus[featureName], int32(l.FeatureStatuses[idx])) columnNameToTimestamp[featureName] = append(columnNameToTimestamp[featureName], l.EventTimestamps[idx].AsTime().UnixNano()/int64(time.Millisecond)) } + strBuilder.Append(l.RequestId) } fields := make([]arrow.Field, 0) @@ -226,6 +230,12 @@ func ConvertMemoryBufferToArrowTable(memoryBuffer *MemoryBuffer, fcoSchema *Sche }) columns = append(columns, arrowArray) } + fields = append(fields, arrow.Field{ + Name: "RequestId", + Type: &arrow.StringType{}, + }) + + columns = append(columns, strBuilder.NewArray()) schema := arrow.NewSchema( fields, nil, @@ -312,7 +322,7 @@ func (s *LoggingService) GetFcos() ([]*model.Entity, []*model.FeatureView, []*mo return entities, fvs, odfvs, nil } -func (l *LoggingService) GenerateLogs(entityMap map[string][]*types.Value, featureNames []string, features []*serving.GetOnlineFeaturesResponse_FeatureVector, requestData map[string]*types.RepeatedValue) error { +func (l *LoggingService) GenerateLogs(entityMap map[string][]*types.Value, featureNames []string, features []*serving.GetOnlineFeaturesResponse_FeatureVector, requestData map[string]*types.RepeatedValue, requestId string) error { // Add a log with the request context if len(requestData) > 0 { requestContextLog := Log{ @@ -358,6 +368,7 @@ func (l *LoggingService) GenerateLogs(entityMap map[string][]*types.Value, featu FeatureValues: featureValueLogRow, FeatureStatuses: featureStatusLogRow, EventTimestamps: eventTimestampLogRow, + RequestId: requestId, } err := l.EmitLog(&newLog) if err != nil { diff --git a/go/cmd/server/server.go b/go/cmd/server/server.go index c5b5996de3b..118831cae4c 100644 --- a/go/cmd/server/server.go +++ b/go/cmd/server/server.go @@ -8,6 +8,7 @@ import ( "github.com/feast-dev/feast/go/protos/feast/serving" prototypes "github.com/feast-dev/feast/go/protos/feast/types" "github.com/feast-dev/feast/go/types" + "github.com/google/uuid" ) type servingServiceServer struct { @@ -30,6 +31,7 @@ func (s *servingServiceServer) GetFeastServingInfo(ctx context.Context, request // Metadata contains featurenames that corresponds to the number of rows in response.Results. // Results contains values including the value of the feature, the event timestamp, and feature status in a columnar format. func (s *servingServiceServer) GetOnlineFeatures(ctx context.Context, request *serving.GetOnlineFeaturesRequest) (*serving.GetOnlineFeaturesResponse, error) { + requestId := GenerateRequestId() featuresOrService, err := s.fs.ParseFeatures(request.GetKind()) if err != nil { return nil, err @@ -72,6 +74,11 @@ func (s *servingServiceServer) GetOnlineFeatures(ctx context.Context, request *s EventTimestamps: vector.Timestamps, }) } - go s.loggingService.GenerateLogs(entityValuesMap, featureNames, resp.Results[len(request.Entities):], request.RequestContext) + go s.loggingService.GenerateLogs(entityValuesMap, featureNames, resp.Results[len(request.Entities):], request.RequestContext, requestId) return resp, nil } + +func GenerateRequestId() string { + id := uuid.New() + return id.String() +} diff --git a/go/cmd/server/server_test.go b/go/cmd/server/server_test.go index c3a1c37769c..2e06256fc67 100644 --- a/go/cmd/server/server_test.go +++ b/go/cmd/server/server_test.go @@ -225,17 +225,23 @@ func TestGetOnlineFeaturesSqliteWithLogging(t *testing.T) { values, err := test.GetProtoFromRecord(rec) assert.Nil(t, err) - assert.Equal(t, len(values), len(expectedLogValues)) + assert.Equal(t, len(values)-1 /*request id column not counted*/, len(expectedLogValues)) // Need to iterate through and compare because certain values in types.RepeatedValues aren't accurately being compared. for name, val := range values { - assert.Equal(t, len(val.Val), len(expectedLogValues[name].Val)) - for idx, featureVal := range val.Val { - assert.Equal(t, featureVal.Val, expectedLogValues[name].Val[idx].Val) + if name == "RequestId" { + // Ensure there are request ids for each entity. + assert.Equal(t, len(val.Val), len(response.Results[0].Values)) + } else { + assert.Equal(t, len(val.Val), len(expectedLogValues[name].Val)) + for idx, featureVal := range val.Val { + assert.Equal(t, featureVal.Val, expectedLogValues[name].Val[idx].Val) + } } + } } - // err = test.CleanUpLogs(logPath) - // assert.Nil(t, err) + err = test.CleanUpLogs(logPath) + assert.Nil(t, err) } // Generate the expected log rows based on the resulting feature vector returned from GetOnlineFeatures. From 86e605db36c07c7747f5aa47611789126bd62b38 Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Tue, 12 Apr 2022 10:14:51 -0700 Subject: [PATCH 72/97] More fixes Signed-off-by: Kevin Zhang --- go/cmd/server/logging/logging.go | 8 ++++++-- go/cmd/server/server.go | 2 +- go/cmd/server/server_test.go | 3 ++- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/go/cmd/server/logging/logging.go b/go/cmd/server/logging/logging.go index d522ca34888..85e2afc7281 100644 --- a/go/cmd/server/logging/logging.go +++ b/go/cmd/server/logging/logging.go @@ -284,8 +284,8 @@ func GetSchemaFromFeatureService(featureService *model.FeatureService, entities featureViewName := featureProjection.Name if _, ok := fvs[featureViewName]; ok { for _, f := range featureProjection.Features { - features = append(features, f.Name) - allFeatureTypes[f.Name] = f.Dtype + features = append(features, GetFullFeatureName(featureViewName, f.Name)) + allFeatureTypes[GetFullFeatureName(featureViewName, f.Name)] = f.Dtype } } else if odfv, ok := odFvs[featureViewName]; ok { for name, valueType := range odfv.GetRequestDataSchema() { @@ -306,6 +306,10 @@ func GetSchemaFromFeatureService(featureService *model.FeatureService, entities return schema, nil } +func GetFullFeatureName(featureViewName string, featureName string) string { + return fmt.Sprintf("%s__%s", featureViewName, featureName) +} + func (s *LoggingService) GetFcos() ([]*model.Entity, []*model.FeatureView, []*model.OnDemandFeatureView, error) { odfvs, err := s.fs.ListOnDemandFeatureViews() if err != nil { diff --git a/go/cmd/server/server.go b/go/cmd/server/server.go index 118831cae4c..e1e7d8a767c 100644 --- a/go/cmd/server/server.go +++ b/go/cmd/server/server.go @@ -58,7 +58,6 @@ func (s *servingServiceServer) GetOnlineFeatures(ctx context.Context, request *s entityValuesMap := make(map[string][]*prototypes.Value, 0) featureNames := make([]string, len(featureVectors)) for idx, vector := range featureVectors { - resp.Metadata.FeatureNames.Val = append(resp.Metadata.FeatureNames.Val, vector.Name) featureNames[idx] = vector.Name values, err := types.ArrowValuesToProtoValues(vector.Values) @@ -68,6 +67,7 @@ func (s *servingServiceServer) GetOnlineFeatures(ctx context.Context, request *s if _, ok := request.Entities[vector.Name]; ok { entityValuesMap[vector.Name] = values } + resp.Results = append(resp.Results, &serving.GetOnlineFeaturesResponse_FeatureVector{ Values: values, Statuses: vector.Statuses, diff --git a/go/cmd/server/server_test.go b/go/cmd/server/server_test.go index 2e06256fc67..7de4d04abe9 100644 --- a/go/cmd/server/server_test.go +++ b/go/cmd/server/server_test.go @@ -192,7 +192,8 @@ func TestGetOnlineFeaturesSqliteWithLogging(t *testing.T) { Kind: &serving.GetOnlineFeaturesRequest_FeatureService{ FeatureService: "test_service", }, - Entities: entities, + Entities: entities, + FullFeatureNames: true, } response, err := client.GetOnlineFeatures(ctx, request) // Wait for logger to flush. From fa14daeb9b99ff06a51e52e3247da7a65ccd3b36 Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Tue, 12 Apr 2022 10:28:17 -0700 Subject: [PATCH 73/97] Fix odfv Signed-off-by: Kevin Zhang --- go/cmd/server/logging/logging.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/go/cmd/server/logging/logging.go b/go/cmd/server/logging/logging.go index 85e2afc7281..a39df36cd98 100644 --- a/go/cmd/server/logging/logging.go +++ b/go/cmd/server/logging/logging.go @@ -288,9 +288,10 @@ func GetSchemaFromFeatureService(featureService *model.FeatureService, entities allFeatureTypes[GetFullFeatureName(featureViewName, f.Name)] = f.Dtype } } else if odfv, ok := odFvs[featureViewName]; ok { - for name, valueType := range odfv.GetRequestDataSchema() { - features = append(features, name) - allFeatureTypes[name] = valueType + for _, f := range odfv.Base.Features { + // TODO(kevjumba) check in test here. + features = append(features, GetFullFeatureName(featureViewName, f.Name)) + allFeatureTypes[GetFullFeatureName(featureViewName, f.Name)] = f.Dtype } } else { return nil, fmt.Errorf("no such feature view found in feature service %s", featureViewName) From 9f81d047d72998952067005d34b0a43ac7a42402 Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Tue, 12 Apr 2022 11:55:07 -0700 Subject: [PATCH 74/97] Fix Signed-off-by: Kevin Zhang --- go/cmd/server/logging/filelogstorage.go | 4 ++-- go/cmd/server/logging/filelogstorage_test.go | 11 +++++++--- go/cmd/server/logging/logging.go | 22 ++++++++++---------- go/cmd/server/logging/logging_test.go | 11 +++++----- 4 files changed, 27 insertions(+), 21 deletions(-) diff --git a/go/cmd/server/logging/filelogstorage.go b/go/cmd/server/logging/filelogstorage.go index fc9119e4a89..19e9569e69d 100644 --- a/go/cmd/server/logging/filelogstorage.go +++ b/go/cmd/server/logging/filelogstorage.go @@ -54,7 +54,7 @@ func NewFileOfflineStore(project string, offlineStoreConfig *OfflineLogStoreConf return &store, nil } -func OpenLogFile(absPath string) (*os.File, error) { +func openLogFile(absPath string) (*os.File, error) { var _, err = os.Stat(absPath) // create file if not exists @@ -70,7 +70,7 @@ func OpenLogFile(absPath string) (*os.File, error) { } func (f *FileLogStorage) FlushToStorage(tbl array.Table) error { - w, err := OpenLogFile(f.path) + w, err := openLogFile(f.path) var writer io.Writer = w if err != nil { return err diff --git a/go/cmd/server/logging/filelogstorage_test.go b/go/cmd/server/logging/filelogstorage_test.go index 44db35e9bc8..172ba323c92 100644 --- a/go/cmd/server/logging/filelogstorage_test.go +++ b/go/cmd/server/logging/filelogstorage_test.go @@ -52,9 +52,14 @@ func TestFlushToStorage(t *testing.T) { assert.Nil(t, err) for name, val := range values { - assert.Equal(t, len(val.Val), len(expectedColumns[name].Val)) - for idx, featureVal := range val.Val { - assert.Equal(t, featureVal.Val, expectedColumns[name].Val[idx].Val) + if name == "RequestId" { + // Ensure there are request ids for each entity. + assert.Equal(t, len(val.Val), 2) + } else { + assert.Equal(t, len(val.Val), len(expectedColumns[name].Val)) + for idx, featureVal := range val.Val { + assert.Equal(t, featureVal.Val, expectedColumns[name].Val[idx].Val) + } } } } diff --git a/go/cmd/server/logging/logging.go b/go/cmd/server/logging/logging.go index a39df36cd98..4e95c04ffba 100644 --- a/go/cmd/server/logging/logging.go +++ b/go/cmd/server/logging/logging.go @@ -27,7 +27,7 @@ type Log struct { FeatureValues []*types.Value FeatureStatuses []serving.FieldStatus EventTimestamps []*timestamppb.Timestamp - RequestContext map[string]*types.RepeatedValue + RequestContext map[string]*types.Value RequestId string } @@ -99,13 +99,13 @@ func (s *LoggingService) processLogs() { defer ticker.Stop() for { - s.HandleLogFlushing(ticker) + s.PerformPeriodicAppendToMemoryBufferAndLogFlush(ticker) } } -// Select that eitheringests new logs that are added to the logging channel, one at a time to add -// to the in memory buffer or flushes all of them synchronously to the OfflineStorage on a time interval. -func (s *LoggingService) HandleLogFlushing(t *time.Ticker) { +// Select that either ingests new logs that are added to the logging channel, one at a time to add +// to the in-memory buffer or flushes all of them synchronously to the OfflineStorage on a time interval. +func (s *LoggingService) PerformPeriodicAppendToMemoryBufferAndLogFlush(t *time.Ticker) { select { case t := <-t.C: s.flushLogsToOfflineStorage(t) @@ -329,12 +329,12 @@ func (s *LoggingService) GetFcos() ([]*model.Entity, []*model.FeatureView, []*mo func (l *LoggingService) GenerateLogs(entityMap map[string][]*types.Value, featureNames []string, features []*serving.GetOnlineFeaturesResponse_FeatureVector, requestData map[string]*types.RepeatedValue, requestId string) error { // Add a log with the request context - if len(requestData) > 0 { - requestContextLog := Log{ - RequestContext: requestData, - } - l.EmitLog(&requestContextLog) - } + // if len(requestData) > 0 { + // requestContextLog := Log{ + // RequestContext: requestData, + // } + // l.EmitLog(&requestContextLog) + // } if len(features) <= 0 { return nil diff --git a/go/cmd/server/logging/logging_test.go b/go/cmd/server/logging/logging_test.go index f9ab351108d..591b3b7f3d2 100644 --- a/go/cmd/server/logging/logging_test.go +++ b/go/cmd/server/logging/logging_test.go @@ -195,6 +195,7 @@ func GetTestArrowTableAndExpectedResults() (array.Table, map[string]arrow.DataTy } expectedSchema[joinKey] = arrowType } + expectedSchema["RequestId"] = arrow.BinaryTypes.String for featureName, featureType := range schema.FeaturesTypes { arrowType, err := gotypes.ValueTypeEnumToArrowType(featureType) if err != nil { @@ -209,22 +210,22 @@ func GetTestArrowTableAndExpectedResults() (array.Table, map[string]arrow.DataTy log1.EntityValue[0], log2.EntityValue[0]}, }, - "int64": { + "featureView1__int64": { Val: []*types.Value{ log1.FeatureValues[0], log2.FeatureValues[0]}, }, - "float32": { + "featureView1__float32": { Val: []*types.Value{ log1.FeatureValues[1], log2.FeatureValues[1]}, }, - "int32": { + "featureView2__int32": { Val: []*types.Value{ log1.FeatureValues[2], log2.FeatureValues[2]}, }, - "double": { + "featureView2__double": { Val: []*types.Value{ log1.FeatureValues[3], log2.FeatureValues[3]}, @@ -256,7 +257,7 @@ func SetupLoggingServiceWithLogs(logs []*Log) (*LoggingService, error) { } // manually handle flushing logs for i := 0; i < len(logs); i++ { - loggingService.HandleLogFlushing(dummyTicker) + loggingService.PerformPeriodicAppendToMemoryBufferAndLogFlush(dummyTicker) } return loggingService, nil } From e87fe221187d705a66b25b19058d6fa8288dfa68 Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Tue, 12 Apr 2022 12:20:01 -0700 Subject: [PATCH 75/97] Fix Signed-off-by: Kevin Zhang --- go/cmd/server/logging/filelogstorage_test.go | 4 ++-- go/cmd/server/logging/logging_test.go | 2 -- go/cmd/server/server_test.go | 13 +++++++++++-- 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/go/cmd/server/logging/filelogstorage_test.go b/go/cmd/server/logging/filelogstorage_test.go index 172ba323c92..319b560e806 100644 --- a/go/cmd/server/logging/filelogstorage_test.go +++ b/go/cmd/server/logging/filelogstorage_test.go @@ -53,8 +53,8 @@ func TestFlushToStorage(t *testing.T) { assert.Nil(t, err) for name, val := range values { if name == "RequestId" { - // Ensure there are request ids for each entity. - assert.Equal(t, len(val.Val), 2) + // Ensure there are request ids in record. + assert.Greater(t, len(val.Val), 0) } else { assert.Equal(t, len(val.Val), len(expectedColumns[name].Val)) for idx, featureVal := range val.Val { diff --git a/go/cmd/server/logging/logging_test.go b/go/cmd/server/logging/logging_test.go index 591b3b7f3d2..deeb5a05f8c 100644 --- a/go/cmd/server/logging/logging_test.go +++ b/go/cmd/server/logging/logging_test.go @@ -28,7 +28,6 @@ func TestLoggingChannelTimeout(t *testing.T) { } loggingService.EmitLog(&newLog) // Wait for memory buffer flush - assert.Eventually(t, func() bool { return true }, 100*time.Millisecond, DEFAULT_LOG_INSERT_TIMEOUT) newTs := timestamppb.New(time.Now()) @@ -38,7 +37,6 @@ func TestLoggingChannelTimeout(t *testing.T) { } err = loggingService.EmitLog(&newLog2) // The channel times out and doesn't hang. - time.Sleep(20 * time.Millisecond) assert.NotNil(t, err) } diff --git a/go/cmd/server/server_test.go b/go/cmd/server/server_test.go index 7de4d04abe9..11fed1360ef 100644 --- a/go/cmd/server/server_test.go +++ b/go/cmd/server/server_test.go @@ -3,6 +3,7 @@ package main import ( "context" "net" + "os" "path/filepath" "reflect" "runtime" @@ -196,8 +197,7 @@ func TestGetOnlineFeaturesSqliteWithLogging(t *testing.T) { FullFeatureNames: true, } response, err := client.GetOnlineFeatures(ctx, request) - // Wait for logger to flush. - assert.Eventually(t, func() bool { return true }, 1*time.Second, logging.DEFAULT_LOG_FLUSH_INTERVAL) + assert.Nil(t, err) assert.NotNil(t, response) @@ -208,6 +208,15 @@ func TestGetOnlineFeaturesSqliteWithLogging(t *testing.T) { expectedLogValues, _, _ := GetExpectedLogRows(featureNames, response.Results[len(request.Entities):]) expectedLogValues["driver_id"] = entities["driver_id"] logPath, err := filepath.Abs(filepath.Join(dir, "log.parquet")) + // Wait for logger to flush. + assert.Eventually(t, func() bool { + var _, err = os.Stat(logPath) + if os.IsNotExist(err) { + return false + } else { + return true + } + }, 1*time.Second, logging.DEFAULT_LOG_FLUSH_INTERVAL) assert.Nil(t, err) pf, err := file.OpenParquetFile(logPath, false) assert.Nil(t, err) From bdf0c2bdea234704400f2b4f1b53823fd4729656 Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Tue, 12 Apr 2022 12:56:43 -0700 Subject: [PATCH 76/97] Address other changes Signed-off-by: Kevin Zhang --- go/cmd/server/logging/logging.go | 69 ++++++++++++++++++-------------- go/cmd/server/server.go | 4 +- 2 files changed, 43 insertions(+), 30 deletions(-) diff --git a/go/cmd/server/logging/logging.go b/go/cmd/server/logging/logging.go index 4e95c04ffba..07821f12209 100644 --- a/go/cmd/server/logging/logging.go +++ b/go/cmd/server/logging/logging.go @@ -123,11 +123,11 @@ func (s *LoggingService) flushLogsToOfflineStorage(t time.Time) error { return fmt.Errorf("could not get offline storage type for config: %s", s.fs.GetRepoConfig().OfflineStore) } if offlineStoreType == "file" { - entities, featureViews, odfvs, err := s.GetFcos() + entities, entityMap, featureViews, odfvs, err := s.GetFcos() if err != nil { return err } - schema, err := GetSchemaFromFeatureService(s.memoryBuffer.featureService, entities, featureViews, odfvs) + schema, err := GetSchemaFromFeatureService(s.memoryBuffer.featureService, entities, entityMap, featureViews, odfvs) if err != nil { return err } @@ -254,26 +254,41 @@ type Schema struct { FeaturesTypes map[string]types.ValueType_Enum } -func GetSchemaFromFeatureService(featureService *model.FeatureService, entities []*model.Entity, featureViews []*model.FeatureView, onDemandFeatureViews []*model.OnDemandFeatureView) (*Schema, error) { +func GetSchemaFromFeatureService(featureService *model.FeatureService, entities []*model.Entity, entityMap map[string]*model.Entity, featureViews []*model.FeatureView, onDemandFeatureViews []*model.OnDemandFeatureView) (*Schema, error) { fvs := make(map[string]*model.FeatureView) odFvs := make(map[string]*model.OnDemandFeatureView) joinKeys := make([]string, 0) - for _, entity := range entities { - joinKeys = append(joinKeys, entity.JoinKey) - } + // All joinkeys in the featureService are put in here + joinKeysSet := make(map[string]interface{}) + entityJoinKeyToType := make(map[string]types.ValueType_Enum) for _, featureView := range featureViews { fvs[featureView.Base.Name] = featureView } - for _, onDemandFeatureView := range onDemandFeatureViews { - odFvs[onDemandFeatureView.Base.Name] = onDemandFeatureView + for _, projection := range featureService.Projections { + fv := fvs[projection.Name] + for entityName := range fv.Entities { + entity := entityMap[entityName] + if joinKeyAlias, ok := projection.JoinKeyMap[entity.JoinKey]; ok { + joinKeysSet[joinKeyAlias] = nil + } else { + joinKeysSet[entity.JoinKey] = nil + } + } } - entityJoinKeyToType := make(map[string]types.ValueType_Enum) + // Only get entities in the current feature service. for _, entity := range entities { - entityJoinKeyToType[entity.JoinKey] = entity.ValueType + if _, ok := joinKeysSet[entity.Name]; ok { + joinKeys = append(joinKeys, entity.JoinKey) + entityJoinKeyToType[entity.JoinKey] = entity.ValueType + } + } + + for _, onDemandFeatureView := range onDemandFeatureViews { + odFvs[onDemandFeatureView.Base.Name] = onDemandFeatureView } allFeatureTypes := make(map[string]types.ValueType_Enum) @@ -311,43 +326,39 @@ func GetFullFeatureName(featureViewName string, featureName string) string { return fmt.Sprintf("%s__%s", featureViewName, featureName) } -func (s *LoggingService) GetFcos() ([]*model.Entity, []*model.FeatureView, []*model.OnDemandFeatureView, error) { +func (s *LoggingService) GetFcos() ([]*model.Entity, map[string]*model.Entity, []*model.FeatureView, []*model.OnDemandFeatureView, error) { odfvs, err := s.fs.ListOnDemandFeatureViews() if err != nil { - return nil, nil, nil, err + return nil, nil, nil, nil, err } fvs, err := s.fs.ListFeatureViews() if err != nil { - return nil, nil, nil, err + return nil, nil, nil, nil, err } entities, err := s.fs.ListEntities(true) if err != nil { - return nil, nil, nil, err + return nil, nil, nil, nil, err + } + entityMap := make(map[string]*model.Entity) + for _, entity := range entities { + entityMap[entity.Name] = entity } - return entities, fvs, odfvs, nil + return entities, entityMap, fvs, odfvs, nil } -func (l *LoggingService) GenerateLogs(entityMap map[string][]*types.Value, featureNames []string, features []*serving.GetOnlineFeaturesResponse_FeatureVector, requestData map[string]*types.RepeatedValue, requestId string) error { - // Add a log with the request context - // if len(requestData) > 0 { - // requestContextLog := Log{ - // RequestContext: requestData, - // } - // l.EmitLog(&requestContextLog) - // } - +func (l *LoggingService) GenerateLogs(featureService *model.FeatureService, entityMap map[string][]*types.Value, featureNames []string, features []*serving.GetOnlineFeaturesResponse_FeatureVector, requestData map[string]*types.RepeatedValue, requestId string) error { if len(features) <= 0 { return nil } - entityArr, err := l.fs.ListEntities(true) + entities, entitySet, featureViews, odfvs, err := l.GetFcos() if err != nil { return err } + schema, err := GetSchemaFromFeatureService(featureService, entities, entitySet, featureViews, odfvs) - joinKeys := make([]string, 0) - for _, entity := range entityArr { - joinKeys = append(joinKeys, entity.JoinKey) + if err != nil { + return err } numFeatures := len(featureNames) @@ -365,7 +376,7 @@ func (l *LoggingService) GenerateLogs(entityMap map[string][]*types.Value, featu } entityRow := make([]*types.Value, 0) // ensure that the entity values are in the order that the schema defines which is the order that ListEntities returns the entities - for _, joinKey := range joinKeys { + for _, joinKey := range schema.Entities { entityRow = append(entityRow, entityMap[joinKey][row_idx]) } newLog := Log{ diff --git a/go/cmd/server/server.go b/go/cmd/server/server.go index e1e7d8a767c..db1102ce204 100644 --- a/go/cmd/server/server.go +++ b/go/cmd/server/server.go @@ -74,7 +74,9 @@ func (s *servingServiceServer) GetOnlineFeatures(ctx context.Context, request *s EventTimestamps: vector.Timestamps, }) } - go s.loggingService.GenerateLogs(entityValuesMap, featureNames, resp.Results[len(request.Entities):], request.RequestContext, requestId) + if featuresOrService.FeatureService != nil { + go s.loggingService.GenerateLogs(featuresOrService.FeatureService, entityValuesMap, featureNames, resp.Results[len(request.Entities):], request.RequestContext, requestId) + } return resp, nil } From f59f98d6694022ad2cbe96fc29f8ebea2ccc4ed5 Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Tue, 12 Apr 2022 13:08:19 -0700 Subject: [PATCH 77/97] Reorder for optimization Signed-off-by: Kevin Zhang --- go/cmd/server/logging/logging.go | 42 +++++++++++++++----------------- go/cmd/server/server.go | 2 +- 2 files changed, 20 insertions(+), 24 deletions(-) diff --git a/go/cmd/server/logging/logging.go b/go/cmd/server/logging/logging.go index 07821f12209..77bb0d4d2ef 100644 --- a/go/cmd/server/logging/logging.go +++ b/go/cmd/server/logging/logging.go @@ -267,26 +267,6 @@ func GetSchemaFromFeatureService(featureService *model.FeatureService, entities fvs[featureView.Base.Name] = featureView } - for _, projection := range featureService.Projections { - fv := fvs[projection.Name] - for entityName := range fv.Entities { - entity := entityMap[entityName] - if joinKeyAlias, ok := projection.JoinKeyMap[entity.JoinKey]; ok { - joinKeysSet[joinKeyAlias] = nil - } else { - joinKeysSet[entity.JoinKey] = nil - } - } - } - - // Only get entities in the current feature service. - for _, entity := range entities { - if _, ok := joinKeysSet[entity.Name]; ok { - joinKeys = append(joinKeys, entity.JoinKey) - entityJoinKeyToType[entity.JoinKey] = entity.ValueType - } - } - for _, onDemandFeatureView := range onDemandFeatureViews { odFvs[onDemandFeatureView.Base.Name] = onDemandFeatureView } @@ -297,11 +277,19 @@ func GetSchemaFromFeatureService(featureService *model.FeatureService, entities // Create copies of FeatureView that may contains the same *FeatureView but // each differentiated by a *FeatureViewProjection featureViewName := featureProjection.Name - if _, ok := fvs[featureViewName]; ok { + if fv, ok := fvs[featureViewName]; ok { for _, f := range featureProjection.Features { features = append(features, GetFullFeatureName(featureViewName, f.Name)) allFeatureTypes[GetFullFeatureName(featureViewName, f.Name)] = f.Dtype } + for entityName := range fv.Entities { + entity := entityMap[entityName] + if joinKeyAlias, ok := featureProjection.JoinKeyMap[entity.JoinKey]; ok { + joinKeysSet[joinKeyAlias] = nil + } else { + joinKeysSet[entity.JoinKey] = nil + } + } } else if odfv, ok := odFvs[featureViewName]; ok { for _, f := range odfv.Base.Features { // TODO(kevjumba) check in test here. @@ -313,6 +301,14 @@ func GetSchemaFromFeatureService(featureService *model.FeatureService, entities } } + // Only get entities in the current feature service. + for _, entity := range entities { + if _, ok := joinKeysSet[entity.Name]; ok { + joinKeys = append(joinKeys, entity.JoinKey) + entityJoinKeyToType[entity.JoinKey] = entity.ValueType + } + } + schema := &Schema{ Entities: joinKeys, Features: features, @@ -346,7 +342,7 @@ func (s *LoggingService) GetFcos() ([]*model.Entity, map[string]*model.Entity, [ return entities, entityMap, fvs, odfvs, nil } -func (l *LoggingService) GenerateLogs(featureService *model.FeatureService, entityMap map[string][]*types.Value, featureNames []string, features []*serving.GetOnlineFeaturesResponse_FeatureVector, requestData map[string]*types.RepeatedValue, requestId string) error { +func (l *LoggingService) GenerateLogs(featureService *model.FeatureService, entityMap map[string][]*types.Value, features []*serving.GetOnlineFeaturesResponse_FeatureVector, requestData map[string]*types.RepeatedValue, requestId string) error { if len(features) <= 0 { return nil } @@ -361,7 +357,7 @@ func (l *LoggingService) GenerateLogs(featureService *model.FeatureService, enti return err } - numFeatures := len(featureNames) + numFeatures := len(schema.Features) // Should be equivalent to how many entities there are(each feature row has (entity) number of features) numRows := len(features[0].Values) diff --git a/go/cmd/server/server.go b/go/cmd/server/server.go index db1102ce204..3708689268b 100644 --- a/go/cmd/server/server.go +++ b/go/cmd/server/server.go @@ -75,7 +75,7 @@ func (s *servingServiceServer) GetOnlineFeatures(ctx context.Context, request *s }) } if featuresOrService.FeatureService != nil { - go s.loggingService.GenerateLogs(featuresOrService.FeatureService, entityValuesMap, featureNames, resp.Results[len(request.Entities):], request.RequestContext, requestId) + go s.loggingService.GenerateLogs(featuresOrService.FeatureService, entityValuesMap, resp.Results[len(request.Entities):], request.RequestContext, requestId) } return resp, nil } From d185ccdd4a25e42acc7bc22749fe15a5c653900e Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Tue, 12 Apr 2022 14:24:30 -0700 Subject: [PATCH 78/97] Fix Signed-off-by: Kevin Zhang --- go/cmd/server/logging/logging_test.go | 63 ++++++++++++++++--- go/internal/feast/model/featureview.go | 8 --- .../feast/model/ondemandfeatureview.go | 2 +- go/internal/test/go_integration_test_utils.go | 9 +++ 4 files changed, 65 insertions(+), 17 deletions(-) diff --git a/go/cmd/server/logging/logging_test.go b/go/cmd/server/logging/logging_test.go index deeb5a05f8c..32ef961703b 100644 --- a/go/cmd/server/logging/logging_test.go +++ b/go/cmd/server/logging/logging_test.go @@ -42,19 +42,38 @@ func TestLoggingChannelTimeout(t *testing.T) { func TestSchemaTypeRetrieval(t *testing.T) { featureService, entities, featureViews, odfvs := InitializeFeatureRepoVariablesForTest() - schema, err := GetSchemaFromFeatureService(featureService, entities, featureViews, odfvs) + entityMap := make(map[string]*model.Entity) + for _, entity := range entities { + entityMap[entity.Name] = entity + } + schema, err := GetSchemaFromFeatureService(featureService, entities, entityMap, featureViews, odfvs) assert.Nil(t, err) assert.Contains(t, schema.EntityTypes, "driver_id") assert.True(t, reflect.DeepEqual(schema.EntityTypes["driver_id"], types.ValueType_INT64)) - features := []string{"int64", "float32", "int32", "double"} - types := []types.ValueType_Enum{*types.ValueType_INT64.Enum(), *types.ValueType_FLOAT.Enum(), *types.ValueType_INT32.Enum(), *types.ValueType_DOUBLE.Enum()} + features := []string{"featureView1__int64", "featureView1__float32", "featureView2__int32", "featureView2__double", "od_bf1__odfv_f1", "od_bf1__odfv_f2"} + types := []types.ValueType_Enum{*types.ValueType_INT64.Enum(), *types.ValueType_FLOAT.Enum(), *types.ValueType_INT32.Enum(), *types.ValueType_DOUBLE.Enum(), *types.ValueType_INT32.Enum(), *types.ValueType_DOUBLE.Enum()} for idx, featureName := range features { assert.Contains(t, schema.FeaturesTypes, featureName) assert.Equal(t, schema.FeaturesTypes[featureName], types[idx]) } } +func TestSchemaRetrievalIgnoresEntitiesNotInFeatureService(t *testing.T) { + featureService, entities, featureViews, odfvs := InitializeFeatureRepoVariablesForTest() + //Remove entities in featureservice + for _, featureView := range featureViews { + featureView.Entities = make(map[string]struct{}) + } + entityMap := make(map[string]*model.Entity) + for _, entity := range entities { + entityMap[entity.Name] = entity + } + schema, err := GetSchemaFromFeatureService(featureService, entities, entityMap, featureViews, odfvs) + assert.Nil(t, err) + assert.Empty(t, schema.EntityTypes) +} + func TestSerializeToArrowTable(t *testing.T) { table, expectedSchema, expectedColumns, err := GetTestArrowTableAndExpectedResults() assert.Nil(t, err) @@ -73,6 +92,9 @@ func TestSerializeToArrowTable(t *testing.T) { assert.Nil(t, err) for name, val := range values { + if name == "RequestId" { + continue + } assert.Equal(t, len(val.Val), len(expectedColumns[name].Val)) for idx, featureVal := range val.Val { assert.Equal(t, featureVal.Val, expectedColumns[name].Val[idx].Val) @@ -102,7 +124,7 @@ func InitializeFeatureRepoVariablesForTest() (*model.FeatureService, []*model.En []*model.Feature{f1, f2}, projection1, ) - featureView1 := model.CreateFeatureView(baseFeatureView1, nil, map[string]struct{}{}) + featureView1 := test.CreateFeatureView(baseFeatureView1, nil, map[string]struct{}{"driver_id": {}}) entity1 := test.CreateNewEntity("driver_id", types.ValueType_INT64, "driver_id") f3 := test.CreateNewFeature( "int32", @@ -123,22 +145,47 @@ func InitializeFeatureRepoVariablesForTest() (*model.FeatureService, []*model.En []*model.Feature{f3, f4}, projection2, ) - featureView2 := model.CreateFeatureView(baseFeatureView2, nil, map[string]struct{}{}) + featureView2 := test.CreateFeatureView(baseFeatureView2, nil, map[string]struct{}{"driver_id": {}}) + + f5 := test.CreateNewFeature( + "odfv_f1", + types.ValueType_INT32, + ) + f6 := test.CreateNewFeature( + "odfv_f2", + types.ValueType_DOUBLE, + ) + projection3 := test.CreateNewFeatureViewProjection( + "od_bf1", + "", + []*model.Feature{f5, f6}, + map[string]string{}, + ) + od_bf1 := test.CreateBaseFeatureView( + "od_bf1", + []*model.Feature{f5, f6}, + projection3, + ) + odfv := model.NewOnDemandFeatureViewFromBase(od_bf1) featureService := test.CreateNewFeatureService( "test_service", "test_project", nil, nil, - []*model.FeatureViewProjection{projection1, projection2}, + []*model.FeatureViewProjection{projection1, projection2, projection3}, ) - return featureService, []*model.Entity{entity1}, []*model.FeatureView{featureView1, featureView2}, []*model.OnDemandFeatureView{} + return featureService, []*model.Entity{entity1}, []*model.FeatureView{featureView1, featureView2}, []*model.OnDemandFeatureView{odfv} } // Create dummy FeatureService, Entities, and FeatureViews add them to the logger and convert the logs to Arrow table. // Returns arrow table, expected test schema, and expected columns. func GetTestArrowTableAndExpectedResults() (array.Table, map[string]arrow.DataType, map[string]*types.RepeatedValue, error) { featureService, entities, featureViews, odfvs := InitializeFeatureRepoVariablesForTest() - schema, err := GetSchemaFromFeatureService(featureService, entities, featureViews, odfvs) + entityMap := make(map[string]*model.Entity) + for _, entity := range entities { + entityMap[entity.Name] = entity + } + schema, err := GetSchemaFromFeatureService(featureService, entities, entityMap, featureViews, odfvs) if err != nil { return nil, nil, nil, err } diff --git a/go/internal/feast/model/featureview.go b/go/internal/feast/model/featureview.go index 4500f3c1d67..bdb86f0ace5 100644 --- a/go/internal/feast/model/featureview.go +++ b/go/internal/feast/model/featureview.go @@ -44,11 +44,3 @@ func (fs *FeatureView) NewFeatureViewFromBase(base *BaseFeatureView) *FeatureVie } return featureView } - -func CreateFeatureView(base *BaseFeatureView, ttl *durationpb.Duration, entities map[string]struct{}) *FeatureView { - return &FeatureView{ - Base: base, - Ttl: ttl, - Entities: entities, - } -} diff --git a/go/internal/feast/model/ondemandfeatureview.go b/go/internal/feast/model/ondemandfeatureview.go index 08db8b416b9..7d2f7a2206b 100644 --- a/go/internal/feast/model/ondemandfeatureview.go +++ b/go/internal/feast/model/ondemandfeatureview.go @@ -48,7 +48,7 @@ func (fs *OnDemandFeatureView) NewWithProjection(projection *FeatureViewProjecti return featureView, nil } -func (fs *OnDemandFeatureView) NewOnDemandFeatureViewFromBase(base *BaseFeatureView) *OnDemandFeatureView { +func NewOnDemandFeatureViewFromBase(base *BaseFeatureView) *OnDemandFeatureView { featureView := &OnDemandFeatureView{Base: base} return featureView diff --git a/go/internal/test/go_integration_test_utils.go b/go/internal/test/go_integration_test_utils.go index 9252c85041f..b0c0205d709 100644 --- a/go/internal/test/go_integration_test_utils.go +++ b/go/internal/test/go_integration_test_utils.go @@ -5,6 +5,7 @@ import ( "fmt" "github.com/apache/arrow/go/v8/arrow/memory" + "google.golang.org/protobuf/types/known/durationpb" "google.golang.org/protobuf/types/known/timestamppb" "github.com/apache/arrow/go/v8/arrow" @@ -192,3 +193,11 @@ func CreateNewFeatureViewProjection(name string, nameAlias string, features []*m JoinKeyMap: joinKeyMap, } } + +func CreateFeatureView(base *model.BaseFeatureView, ttl *durationpb.Duration, entities map[string]struct{}) *model.FeatureView { + return &model.FeatureView{ + Base: base, + Ttl: ttl, + Entities: entities, + } +} From 3747a5d4c61e475e3460b3e4faec852bbbedb78a Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Tue, 12 Apr 2022 14:52:43 -0700 Subject: [PATCH 79/97] Add more shcema tests Signed-off-by: Kevin Zhang --- go/cmd/server/logging/logging.go | 4 +- go/cmd/server/logging/logging_test.go | 80 ++++++++++++++++++++++++++- 2 files changed, 79 insertions(+), 5 deletions(-) diff --git a/go/cmd/server/logging/logging.go b/go/cmd/server/logging/logging.go index 77bb0d4d2ef..ebf05292b1b 100644 --- a/go/cmd/server/logging/logging.go +++ b/go/cmd/server/logging/logging.go @@ -290,8 +290,8 @@ func GetSchemaFromFeatureService(featureService *model.FeatureService, entities joinKeysSet[entity.JoinKey] = nil } } - } else if odfv, ok := odFvs[featureViewName]; ok { - for _, f := range odfv.Base.Features { + } else if _, ok := odFvs[featureViewName]; ok { + for _, f := range featureProjection.Features { // TODO(kevjumba) check in test here. features = append(features, GetFullFeatureName(featureViewName, f.Name)) allFeatureTypes[GetFullFeatureName(featureViewName, f.Name)] = f.Dtype diff --git a/go/cmd/server/logging/logging_test.go b/go/cmd/server/logging/logging_test.go index 32ef961703b..975425cb811 100644 --- a/go/cmd/server/logging/logging_test.go +++ b/go/cmd/server/logging/logging_test.go @@ -1,6 +1,7 @@ package logging import ( + "math/rand" "reflect" "testing" "time" @@ -43,17 +44,35 @@ func TestLoggingChannelTimeout(t *testing.T) { func TestSchemaTypeRetrieval(t *testing.T) { featureService, entities, featureViews, odfvs := InitializeFeatureRepoVariablesForTest() entityMap := make(map[string]*model.Entity) + expectedEntityNames := make([]string, 0) + expectedFeatureNames := make([]string, 0) for _, entity := range entities { entityMap[entity.Name] = entity + expectedEntityNames = append(expectedEntityNames, entity.Name) } + for _, featureView := range featureViews { + for _, f := range featureView.Base.Features { + expectedFeatureNames = append(expectedFeatureNames, GetFullFeatureName(featureView.Base.Name, f.Name)) + } + } + for _, featureView := range odfvs { + for _, f := range featureView.Base.Features { + expectedFeatureNames = append(expectedFeatureNames, GetFullFeatureName(featureView.Base.Name, f.Name)) + } + } + schema, err := GetSchemaFromFeatureService(featureService, entities, entityMap, featureViews, odfvs) assert.Nil(t, err) - assert.Contains(t, schema.EntityTypes, "driver_id") + + assert.Equal(t, expectedFeatureNames, schema.Features) + assert.Equal(t, expectedEntityNames, schema.Entities) + for _, entityName := range expectedEntityNames { + assert.Contains(t, schema.EntityTypes, entityName) + } assert.True(t, reflect.DeepEqual(schema.EntityTypes["driver_id"], types.ValueType_INT64)) - features := []string{"featureView1__int64", "featureView1__float32", "featureView2__int32", "featureView2__double", "od_bf1__odfv_f1", "od_bf1__odfv_f2"} types := []types.ValueType_Enum{*types.ValueType_INT64.Enum(), *types.ValueType_FLOAT.Enum(), *types.ValueType_INT32.Enum(), *types.ValueType_DOUBLE.Enum(), *types.ValueType_INT32.Enum(), *types.ValueType_DOUBLE.Enum()} - for idx, featureName := range features { + for idx, featureName := range expectedFeatureNames { assert.Contains(t, schema.FeaturesTypes, featureName) assert.Equal(t, schema.FeaturesTypes[featureName], types[idx]) } @@ -74,6 +93,61 @@ func TestSchemaRetrievalIgnoresEntitiesNotInFeatureService(t *testing.T) { assert.Empty(t, schema.EntityTypes) } +func TestSchemaUsesOrderInFeatureService(t *testing.T) { + featureService, entities, featureViews, odfvs := InitializeFeatureRepoVariablesForTest() + expectedEntityNames := make([]string, 0) + expectedFeatureNames := make([]string, 0) + entityMap := make(map[string]*model.Entity) + for _, entity := range entities { + entityMap[entity.Name] = entity + } + for _, entity := range entities { + entityMap[entity.Name] = entity + expectedEntityNames = append(expectedEntityNames, entity.Name) + } + // Source of truth for order of featureNames + for _, featureView := range featureViews { + for _, f := range featureView.Base.Features { + expectedFeatureNames = append(expectedFeatureNames, GetFullFeatureName(featureView.Base.Name, f.Name)) + } + } + for _, featureView := range odfvs { + for _, f := range featureView.Base.Features { + expectedFeatureNames = append(expectedFeatureNames, GetFullFeatureName(featureView.Base.Name, f.Name)) + } + } + + rand.Seed(time.Now().UnixNano()) + // Shuffle the featureNames in incorrect order + for _, featureView := range featureViews { + rand.Shuffle(len(featureView.Base.Features), func(i, j int) { + featureView.Base.Features[i], featureView.Base.Features[j] = featureView.Base.Features[j], featureView.Base.Features[i] + }) + } + for _, featureView := range odfvs { + rand.Shuffle(len(featureView.Base.Features), func(i, j int) { + featureView.Base.Features[i], featureView.Base.Features[j] = featureView.Base.Features[j], featureView.Base.Features[i] + }) + } + + schema, err := GetSchemaFromFeatureService(featureService, entities, entityMap, featureViews, odfvs) + assert.Nil(t, err) + + // Ensure the same results + assert.Equal(t, expectedFeatureNames, schema.Features) + assert.Equal(t, expectedEntityNames, schema.Entities) + for _, entityName := range expectedEntityNames { + assert.Contains(t, schema.EntityTypes, entityName) + } + assert.True(t, reflect.DeepEqual(schema.EntityTypes["driver_id"], types.ValueType_INT64)) + + types := []types.ValueType_Enum{*types.ValueType_INT64.Enum(), *types.ValueType_FLOAT.Enum(), *types.ValueType_INT32.Enum(), *types.ValueType_DOUBLE.Enum(), *types.ValueType_INT32.Enum(), *types.ValueType_DOUBLE.Enum()} + for idx, featureName := range expectedFeatureNames { + assert.Contains(t, schema.FeaturesTypes, featureName) + assert.Equal(t, schema.FeaturesTypes[featureName], types[idx]) + } +} + func TestSerializeToArrowTable(t *testing.T) { table, expectedSchema, expectedColumns, err := GetTestArrowTableAndExpectedResults() assert.Nil(t, err) From e9bd35babf9ebbc53c7aab9778827cfc74df0280 Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Tue, 12 Apr 2022 15:03:33 -0700 Subject: [PATCH 80/97] Fix tests Signed-off-by: Kevin Zhang --- go/cmd/server/logging/logging_test.go | 42 ++++++++++++++++++++------- 1 file changed, 32 insertions(+), 10 deletions(-) diff --git a/go/cmd/server/logging/logging_test.go b/go/cmd/server/logging/logging_test.go index 975425cb811..c7b7fdf40ec 100644 --- a/go/cmd/server/logging/logging_test.go +++ b/go/cmd/server/logging/logging_test.go @@ -270,19 +270,25 @@ func GetTestArrowTableAndExpectedResults() (array.Table, map[string]arrow.DataTy {Val: &types.Value_Int64Val{Int64Val: 1001}}, }, FeatureValues: []*types.Value{ - {Val: &types.Value_Int64Val{Int64Val: 1000}}, - {Val: &types.Value_FloatVal{FloatVal: 0.64}}, - {Val: &types.Value_Int32Val{Int32Val: 55}}, - {Val: &types.Value_DoubleVal{DoubleVal: 0.97}}, + /* normal feature values */ + {Val: &types.Value_Int64Val{Int64Val: rand.Int63()}}, + {Val: &types.Value_FloatVal{FloatVal: rand.Float32()}}, + {Val: &types.Value_Int32Val{Int32Val: rand.Int31()}}, + {Val: &types.Value_DoubleVal{DoubleVal: rand.Float64()}}, + /* odfv values */ + {Val: &types.Value_Int32Val{Int32Val: rand.Int31()}}, + {Val: &types.Value_DoubleVal{DoubleVal: rand.Float64()}}, }, FeatureStatuses: []serving.FieldStatus{ serving.FieldStatus_PRESENT, serving.FieldStatus_PRESENT, serving.FieldStatus_PRESENT, serving.FieldStatus_PRESENT, + serving.FieldStatus_PRESENT, + serving.FieldStatus_PRESENT, }, EventTimestamps: []*timestamppb.Timestamp{ - ts, ts, ts, ts, + ts, ts, ts, ts, ts, ts, }, } log2 := Log{ @@ -290,19 +296,25 @@ func GetTestArrowTableAndExpectedResults() (array.Table, map[string]arrow.DataTy {Val: &types.Value_Int64Val{Int64Val: 1003}}, }, FeatureValues: []*types.Value{ - {Val: &types.Value_Int64Val{Int64Val: 1001}}, - {Val: &types.Value_FloatVal{FloatVal: 1.56}}, - {Val: &types.Value_Int32Val{Int32Val: 200}}, - {Val: &types.Value_DoubleVal{DoubleVal: 8.97}}, + /* normal feature values */ + {Val: &types.Value_Int64Val{Int64Val: rand.Int63()}}, + {Val: &types.Value_FloatVal{FloatVal: rand.Float32()}}, + {Val: &types.Value_Int32Val{Int32Val: rand.Int31()}}, + {Val: &types.Value_DoubleVal{DoubleVal: rand.Float64()}}, + /* odfv values */ + {Val: &types.Value_Int32Val{Int32Val: rand.Int31()}}, + {Val: &types.Value_DoubleVal{DoubleVal: rand.Float64()}}, }, FeatureStatuses: []serving.FieldStatus{ serving.FieldStatus_PRESENT, serving.FieldStatus_PRESENT, serving.FieldStatus_PRESENT, serving.FieldStatus_PRESENT, + serving.FieldStatus_PRESENT, + serving.FieldStatus_PRESENT, }, EventTimestamps: []*timestamppb.Timestamp{ - ts, ts, ts, ts, + ts, ts, ts, ts, ts, ts, }, } @@ -349,6 +361,16 @@ func GetTestArrowTableAndExpectedResults() (array.Table, map[string]arrow.DataTy log1.FeatureValues[3], log2.FeatureValues[3]}, }, + "od_bf1__odfv_f1": { + Val: []*types.Value{ + log1.FeatureValues[4], + log2.FeatureValues[4]}, + }, + "od_bf1__odfv_f2": { + Val: []*types.Value{ + log1.FeatureValues[5], + log2.FeatureValues[5]}, + }, } loggingService, err := SetupLoggingServiceWithLogs([]*Log{&log1, &log2}) if err != nil { From 89974b3908025a8bd4fdad9c8c855bb316320954 Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Tue, 12 Apr 2022 15:14:16 -0700 Subject: [PATCH 81/97] refactor to clean Signed-off-by: Kevin Zhang --- go/cmd/server/logging/logging.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/go/cmd/server/logging/logging.go b/go/cmd/server/logging/logging.go index ebf05292b1b..c8dd0943f8f 100644 --- a/go/cmd/server/logging/logging.go +++ b/go/cmd/server/logging/logging.go @@ -342,7 +342,7 @@ func (s *LoggingService) GetFcos() ([]*model.Entity, map[string]*model.Entity, [ return entities, entityMap, fvs, odfvs, nil } -func (l *LoggingService) GenerateLogs(featureService *model.FeatureService, entityMap map[string][]*types.Value, features []*serving.GetOnlineFeaturesResponse_FeatureVector, requestData map[string]*types.RepeatedValue, requestId string) error { +func (l *LoggingService) GenerateLogs(featureService *model.FeatureService, joinKeyToEntityValues map[string][]*types.Value, features []*serving.GetOnlineFeaturesResponse_FeatureVector, requestData map[string]*types.RepeatedValue, requestId string) error { if len(features) <= 0 { return nil } @@ -370,13 +370,13 @@ func (l *LoggingService) GenerateLogs(featureService *model.FeatureService, enti featureStatusLogRow[idx] = features[idx].Statuses[row_idx] eventTimestampLogRow[idx] = features[idx].EventTimestamps[row_idx] } - entityRow := make([]*types.Value, 0) + valuesPerEntityRow := make([]*types.Value, 0) // ensure that the entity values are in the order that the schema defines which is the order that ListEntities returns the entities for _, joinKey := range schema.Entities { - entityRow = append(entityRow, entityMap[joinKey][row_idx]) + valuesPerEntityRow = append(valuesPerEntityRow, joinKeyToEntityValues[joinKey][row_idx]) } newLog := Log{ - EntityValue: entityRow, + EntityValue: valuesPerEntityRow, FeatureValues: featureValueLogRow, FeatureStatuses: featureStatusLogRow, EventTimestamps: eventTimestampLogRow, From d51544bb2ebd977ed794545e6f360608671e1002 Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Tue, 12 Apr 2022 18:17:56 -0700 Subject: [PATCH 82/97] Add initialized repo for testing Signed-off-by: Kevin Zhang --- .../server/logging/feature_repo/__init__.py | 0 .../feature_repo/data/driver_stats.parquet | Bin 0 -> 34704 bytes go/cmd/server/logging/feature_repo/example.py | 40 ++++++++++++++ .../logging/feature_repo/feature_store.yaml | 5 ++ go/cmd/server/logging/logging_test.go | 2 - go/cmd/server/server_test.go | 50 ++---------------- go/internal/test/go_integration_test_utils.go | 30 ++++++++++- 7 files changed, 78 insertions(+), 49 deletions(-) create mode 100644 go/cmd/server/logging/feature_repo/__init__.py create mode 100644 go/cmd/server/logging/feature_repo/data/driver_stats.parquet create mode 100644 go/cmd/server/logging/feature_repo/example.py create mode 100644 go/cmd/server/logging/feature_repo/feature_store.yaml diff --git a/go/cmd/server/logging/feature_repo/__init__.py b/go/cmd/server/logging/feature_repo/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/go/cmd/server/logging/feature_repo/data/driver_stats.parquet b/go/cmd/server/logging/feature_repo/data/driver_stats.parquet new file mode 100644 index 0000000000000000000000000000000000000000..a1e196df26ec4d58205bf198870dadb2e8e595bf GIT binary patch literal 34704 zcmb5Vc|29o`}d8^bD1Sc<|&ynoPAy9nIxH$gpiO*NM*`Qrp%!d5<+EaLdcXfQb`&} zC210x?$v#N@7MEt?&0-%9{;?L^Ld|hUi&P2?`vIau`{sWD8ML?sVp!iV=Zu>Nq}(` z9bKQG5}n)?eye9pbaX@JADCD3KQ%Lkeu@Svgx*1ltn0=bLOMpi>mHqi1EU zxbcOE=Jx$th&Ve{W>2Ebh_I~$8eBFG1Y_mvw-GT)W0{B&eIt%U>~^biB9Y&xBhCbz zaSkp74T6fU1P(RyZp2@t#)w~Xuil;b`aZ|EljzN&n+HKsn%WKmB?&f9B92^oLBs=Q z%{z%G{4C9jMAw3L?IQ4G(Df!@%;WVTVzvBRB3fU+?n}gZyWHI*%6${yN1z>U;7>5l zAsj%&=(0H?N~!nkA!1)|aUh8TT|@U0@XnbB5nPU08%#uJfdwM|svZgIt0d#!ntU^0yJ7!l*yUlUQTv@M#5LmM+=NEFiJ8%x0N zLd6lZ&ho_*(LMS-5$X6lj}x(?DnEfl=6XSi1hd1&Nd%c*tCNYSvGAFQW66CfM2uQf zmP#Ut#)D}D-Nx4G1b$=#W#HNuouWBUuC`?OU@6^lcS#2;RJ; z%O!r&!BOHXvDKd>{zyrD9*GX9y5$oHcW*2pxaP!KNJP&M6GUV@cBP1j)vMBqNn~B= zRYEYYqjQQN_YO}f5w&+t6LI|8wK5_`C*_orNJ=c=G(q1*{WAoCMnY$a$UFLxh?fJp zD~Ra)r|2Aser4^eBq*0QuOcvR6R#%XjP-XSrcDl>C!*?sGc_c7$P#gZAiQ{UErHku znK~kNbp0Ztx8s9)BC@_ef00DBN1_`D>{dE85`3#rZXzhqX1qjv+R#(tPwr^EO#Jw- ziC0J@m*~+e4&i!vdjnCYAPQ)3_q6reE z^@qG5P~C1eN$_xf%}XMN$A2TDnDF2$B6eIj{hCDH82*NU^?~gaL9MU!G!gBVe-iOq z`u(>=ELeAbhD5Y0$KDZ4nmfKHh#yz_Kp+>gVwU(rEB*~=NPfdd;`1ve&XK6~#`aGH z?)Dn<1a#BvpNUuz@sfz{7cU&mt?k% zZk$}jm7e_|=19;mJuwe3oMRx0aNgk+1lQ#483{bE%Q2C+WZd$Xh}CZ%F%!``yq1MT z^Bi%k1i57{Yy{fss_X>gz04fMk9K`de5tuhoW$>oN#P<4BuNQL?SWz5MhFj zTvHJOZ@D$1L}b18m58;Q2UZi&?sd5siN1v$T0>C4ZYxeeE0tPH#L12SMl`7Kuij_HQCM@WDcxK=_!14iT@d`Zu7S zm3MTB$f#SPN22OGku(D9om(h^`ERm7#N4DmMAR00ghU*_ctM{;(MGWb1X82Uh6H^9 z8;l47|1cR7pErA)_?M+GnGoN(E!mVrzpOpY2+Ak5%n6JSa9I#>hUMRYrWLna5>a(S zmKBK}cI~z%2zLY<0_}9r9lV9WZpg%*;M)$dtwb#N z`mchZCH6ZIaZQwV0kl4nyi-_Gb|0)Q- zsAg{>a`LA8kf`D8E?)wNO}e`Y76*9!h*;wBuY%D3a^0VZZ;s~%kSIxH&mID$I)gxh z5q;skL_F~DUj-rT*AqmgKhfM6|a2S3%4_AKFjE+hS>~RiZHNUZr({YJB5{5~_24~`v`Q+@YojcDQPZd)2>MJlfn~=U*J9K8O^K9aY zwfeEFB8C-7S?kTSdrYqkA(*Pfb_7n~Zr`io31+Ib%_5T32*s&sdZHv);Pmmf*>*NC2jj>?w@zNU4QE8q4`(+ z#by^vuOD9g^6uH~i)A;D(XC(?GjAyGjAP|faPMn4eJhc7y=94cT0h*>mM^ykQJt#R*fIyacFx@(}s;!@?EV(tC!$NMi;-7V9PV_#!=x%%E&^Bjfk z1DDS~sIoh2dCKxi&7%v>mja#-T)8lM(eqZy8ms2oCzt%_8F^Ks_32lz@JiV=L>sJN z;gK-TRE;rY;t*2x9%+a%V&Pe>?G1i1bXoF{W=EgWvuGO1D-i$QH znenW}XvURl@#ZVnn`KIEX^OYtligfyoTYZ$Qb5tE!TVm*aVwz>J38W5swY^BsP7)B z*m5bsX7#4v$?h!mMB6obhvwhhyOg+jEgqv|TBVU>x6UY$SK9t^(w6mR$4*F?WNReb zOIziu`rN;qyj6B{nf~!rnkf$QTdVBO*%*-n4J}VO!4b&1D$FETEg~x8q_An@^jb)IAxn?SfOI@1ypUdz>!r=u8mM%L{b5wEG?dtArpCySsfom{J8*JUWur?P5+B)e4&I zJH3nLgg6|fW0Uk|&5|_Dkgz!8&6Xo$TdJQd>BD|f!Li=6|BMeuzKX| z!{rudeYuLYf+jkW*YD;&r5iRk*?)F7PZ=EjL&q-V$9vi^fqR{0h2P4vrWxV}DN_Ec z&ROQE>>Q}@=c}?Ug&20}0RHp#mA0oX&jkovaIE*}Op)FrSm)Xt^m5?b9-)gKH;&SC z$OH;Edi7?kv#Jafx#T-sYLF_kSM*B2ghn|#35}mQkJd)PcA?t1>aIZB?liMfRlOnjmYp8nP z`r8@D#4(mrK2)kdTW0ea>+_+~gLx`Dy3*wL%iJl_3Vt7@6@19}e)$gEVVB!?h&(!Z^ zVCE8|Me`bTF|rFNxp!n5++yZlLyMwYY1qxWO4_=7o%eNE$&NM zo>bc2m22@p#`YBL2!nv-Lpl3%)~5_lT0T;6taHB}D`53l$@NO$^RAOtqbeTPqkc08 zT92uE^`xya%Cml==6k1viW9VXsuA$0cKfY7n`c@<&#xU>A!IwQ9Xd61$|&FVxo+6U z@#}FynghZ(ywCh6iP zMGo)mD{W4}anWt>9cs4KdGr)*``}o={rXX+)sC~yO+I%@O*kZ?qv;vhg|(w4Vq%#% zMYY4}*2TuN@hdpDMz4!Y;1bgfU!;?aPg*HuS)d&wc|1ix$+b0{etklkh(^FzYs~t@ zj5V~##YK9lq|9|z#wo%&u~Ny|($+=V5e(8Px$+J*&TX;Msd>uo?cqxdGHC@Hy$1?( z;$+f`Gy|WuMy!y{DA76aeylA{_CzTawY<2rLN2r1Af8=BH(oC5j7gfPP9&pzc7;W* zf=he6d`_iJiEhL)qe5==mI}*4-Qx-;Yqr(8wnZ{2=GD4f4tUajTrt0X`?biWWhSM9 zhMnChB6m5VO>Kdf=NnxI_Vym!1k;wQ68N$b8h1BH5tDyP~H&OU8B z%(9{MTEw^aPp&3zD7$gw*YeU&7S-~O7)B0JT9WGNuH&4mb)#4}p6O2JS9HCWwDD|j zx|m+%FIKgRzAP!LB3iQAxq*{PZtYQQ>Xk!<8hf5zOIEKMK1Dmc{EJPadgQcmswkDB zasK`}>tfv_?3y(X&pTXjy`G|Z;c=b&)yUuMTD4=1-h)L{YM!7nolKHBDIG*K8R?is zgn4|ze1Z<_IN-n6Xa9kndqV^m1WUyP>um+2&1GU`Wmy0JR2L#jq#|JyVE9+0`~%%T z*a_%c3$Xw9K9d~l#}#x;^mMaataKl^SZVM7)gk2P-G4}4rY4~E|Eg@r3;z3$Fzf%6 z{#7>r`?Ew6@%-nKvS!77Cc67f^!J$x7PJw)pPxNzmYX21)t0C@M*Z8?^Y)dipK6GKS&RJX!ql$O1!EGN{I$D7>*P5;@FO za2>N4SkeCh_7W~o5L<$hq7YOM6~n(jwqmxTG};;cfDt}B9Lm{^F&;+fDX<>z<*{Ob zehKI*`%vi*3{Wac3Ka*OajiUsyGykp?iC*u>&%bRG&P)f<&Im08>l7Ir?Bkoj|G%1 z-k1(T`vnECP1S({@duQ#AsyCdufj6tX82RK9izu@f!PIaT%#0@Tt-K*(IOQXTsdfI zCnZs-eGy)?bW$&>qo_Z(qwufuEQIv$!#8;}W5*jp7I+XTze`7z6Pn&gbaC5Ei?kT4t#=|y=9=yDvu_IRH+L) z4`9G2D>TyFjd^_EA=F(5CqiBEPK*iE^0=bUeI*S0;(&ep$1r--3F(JaP-YJ&6pd=4 zf>I@niv?j-90LX`TjBhg%hYQQUaSuQ6nvrtEU`S8ikj5Cq zjel%b!sKNkoX^q%WBow9`^yav`_b?LOBrnY&Wm@&?!vpk2DoqC4-yBgux88yZ@0SR zJ%4t5!fS-r?g@aq*BBh%yAIhWAAo#Z7no;=P@cbnsVSd%>K79~nj6a@y$3%w@A3wX zPJI;66NAiGyf~ij3tx}C1IZgZsBQiZ7~kCl7x&QNl~ewB|BwoPDDVceUF*@2IScxH z6i}+g4Gj!r;NYgsIPlH}SJwUpM+Xl)u-XEPn*PEH!E+F6^Nd=THUtu1&%o1*!g>!Q zpMyb{9L9T?qn!j76?O5rUcG51!a#!iPe5{60~y%O?%8Dn0RVoD#S;nBt*EL5%13LvHRp z=&Kls>s~j5iA@j0yS#&`0ygxJN&^`#ZJ0OWMd61BK>KnU<;H7;!~56Z+ig1`>4Ye{ z-8cy94IR|eBZ4@!_cMqpB*K2fRXBVi0Q1#Zpu;8v7GJDHqZL~z?!O^;HiirHogL6H zUII_bc~e<$l4#98?J!MvJ$ltC;h{HcFlMTrqED4TODi3u@o&T9J5+I9S3D)N;0qRc zQLxg{1$O4xW8%j~YG2McbjONd-ph4(baybG&R&nq<(pBc=>TdqX~MMfcZ%IU6q!Cd z;;-GzxM5fu4bR!Z!s=e|&>4rY4Qd$K&`Jd)o`olP0lrH5p_W=O*g4bVl)M@CJlKQ1 z2janUqcOyc=0F%rBCL2g2NExJ(1{90Q)zl!mg7Q=WCz@<#ej#oOi@%^8=F5J#~_aF{h7=F%q~#Ig`5PQ3CS!_ux)EhHN*rQ7_sPJKG#FqJ{jOq;q2Q zq6)lN*n({sfiZIeW!#IJ$Aj=<wGjZ)ZX>5H9C@2C|OUYKfXhI@)tFz^c-22}rm+`vINa!n5ze$b;` zk}A063B#cc68K5P0KXNn;Ifnp9z;(pbSi|($Yf}{Vt^uVJmI{ zjZXLz9D-MRcVc5;0mU!#5~5vesp${~{CPK)+W$TPq3ApG7&k({6erFu*T9x9pCMj) zB_q=QS+x_o@Q8ziw>Z`_v4N0-Z*kr0^RB9;jg*_2rvfY(zbBm z-DC=`2AZfW-VYzYy5Z1wHEQk_175yV3YueE@WOX`)H=w3CXFr_k>!N*MwU1{rhuKF zL*ZtS25hL`gocLp$j{`4{W8f=zIQ8luCW4nHVgdKa}7kVjzQ89MNIg95YC-hjWM!U zp!1w5@>u1=aJL53%?P2z)&y$fa2UqFD1_X*j=1^G<>2x7B!g`WSQ`;=pK+Fgzh$3O2GEka?3f z9yd0`Nrnedm2@3CmzAKT<|wXK*pHi(Za^lJKT3aKfZ6a+OxU;$nWkmXyM+x^x4U9| z|2FDLLnC!tvm5paxng@Oebl(}R8*<+R7wY`*>l8coDV9Q+z73@qs|U|cB*X1DGI;pLP1qh+2M0vT zL0wh_c`L~0>`}opZ*=h4^b^Wu{2U-(JnWp6Ms?qd@WJRURrHe`<7(tl&t(x-NLgU% z=@0Pdz+o&Z3IVlR7If4x#i#ps;7_Z&Pc4ugENyVK zDG5Gb=R}6!a1>o$gj;{P!642XCs*&ps^raRJ?M^W?uCQV_tW4U>WF_5YAH3&TQJ6J zhjVPoSn+3+>il*To%wXp-fkO4jGv{Vj59&#_bM}X>L_CuO%4;sDyU1&f)E)UfbT3Cpe*_v7^VjSbNL?X(WO_g;_C&y zKdA+f?XHMvdudpAct84awLw?xc1#vZ(~C9|$BbzatZcWzTDcl{S#cUl9|IosV8wUk zb+E=E5>6Ft#KSEI@a;7bd}bksnM}C5+geH#aTch#KU3hNS9O$n^ z@bKJDrLYKMi2Z)>uQkQLzmMQR6F+wSDS|_;GPq7p7Uj*m;FtrcKjZd+SNk5y^YkXH zTeu9bnK%$~U&DQMPMH5rt}8q{@RaUKxUg>sUev2#f4mT`G@FIJ*_K!sW{+5Bf*}Dq z*dZr?pB%Z+UEw{j|5Cv1^xEiN)C2Z6RHzzLPB`UJ1$TfGHJwUnAFIAXE4>oZYZk+p z)PCehFhrx+InYz4VRcqK?Ak4jl|_rdHo}2|b?SI!F&3&zLr^bD7%xr?QF}hWq;5T$ z1ncz_vaNDKlWYULy?6wp-aLhh)5p+ghy{n(Ye7v=3^hj7{C zAq2LB!LidC=%L7gQ*w8x6n---U7Dq8-@0JtXKO6lx*DrB9_g{~zfJ2P_h>Z%#;+sa+ZxyyWJkw-%W-EZx__5*ntIZv2ee<9(bSdu9_@;)Aeio?kx&n^x zbj7hPZn)_=2TIPkqsH(-92u6u8xI#~Q=h%S_plr+eXs>OIt#?MNs8&P3i7NDLjyBm z-1XuotS#CD7Hosm#0f^!A=fL|XM)Y1?AVZIhfDFlLFRffJP_BzNpoT3&Um25#-sts zpDpmJv;+Rxgs|wQj*I43sg_vxThw#S$Bcw?`grJZbSTUhU*5r#elBibIq(U)7WZ;u%Ys>tE9Xja_LY+(%;MM5|(+;)WPc8l=fn+m=;sRspmS+FE(g}v?(*rXwjBV~zj zQ~WEuu5m$Txp~<3g9mPkEP`b$$3rgNn5(j_6@7g`sMB53e_grb}Tq<}&N(Jr9Lg6L<2HZBN13T?R zP~!|am-h~VbCeoX1=--sKus8QrlIQ*J$#hR0?&BFAV}K)B;g~>ToZ;Dy&Sk}9VZT{ zp8!X@t@wS12p)So0X%#DfP39RR8!-}fFv)pKlqp0$jpws;$$w`%z!V;ol)q}4cg8J z8dP3DAj*qL;PbyhxPI^$6+E*O*GxE333pm(51sY{&&+=KWw#%5CLV(_xlY>ZUxWTW zHEbHbLfzaOj?e3|K)=8kPagaWJBJcsMDh@-8-0aOmfk4U)JJ*R%HsT>3+UNj1-@Pn zWG=V_)Ic;`rCW>Rz4eq-+7IY2;-MtB?Zy=yEAZ=ucTjm(AB}DGQC344k4dUxAS%MX z{0-D2sX#FGj)OWG>(z ze-y_~v*4P(FTm7W1mf|2cw&hY9vrv_j|Vtl8ZhMw zKvo_xv>h0QpaYhu*FlHpbqujEgdX=PQrI(1>MNIb)K-}{wBR9OR7#kJ7kjl(Ec_MB zOmX0)Vj*Pll>?rM7>dur7XnnoF*J!6Yet#TJohT-?%>4l*(~^M@h_F?d<(ecT+vn0 zA2mF$!4l<%C4a<`)%X^;Do<0NT0Cf9r;ea9FBeEj$HE~6Ei%`<2WQo+Fl>b;?q)GW z&kS*NFIGieCrR|#b{_c64-3hV*t>tb4A6;i=ZRdf6Ba<;x%f7ksx{v_QNCodDxwQ6g`7Y@zaYg zXwgrEds}W%uTsvzh0Kq%9o2_XdsQUZZ#2Z;O0OYlZWO+Ee}qvlUdVAgL$SZxj#m;m z@qv*a^?kaOmQlPHr`E25rTh9=_WmigYhD-~e3Ve!Q5w#_n1+BLf6ROth3&k2m{M~T zRF|ZnKFS;HY^?FzdqkBbQ7GON0fm-NLDwMxwEX#yucn=nuo|b#K1L#QFcVB`AoT)uX>U43Sy6l-160I9hTE zTK8~bXI>lFUXsV_`}}e6DI*je_yX0OPhdc`1gS&%sQeN(m92zs zrt`3w%OC86x1rOU6rfWmhZ*%M_;A<^SeGSWh*HNBlE!HCl!h%QHbA)deJG+#;ALk% zocy{L4Fc1F&O{I|t4&efAN26tRdf9M^%iWe$pvQLP%KlIhq?Y8c$+~VGvBi#t5p!n z;s$*7iNczd!bm;(4E=&0c%F5V(p)_O*6c3GYaoxoI}8yTKkI$7>xAX{-FWh}C3cB% z!;c-C(d|kd^~^LItXp4zzE==+BGVY}S#3ehN6ygp-Uow|8sTeT3>0ry#2u^U(CEAn zs>}C+y_P@buZcpFJ9J1jis9a!{gmT2VLbor1@(i@0VcfEK`mnu#w1R{<<1?LVX_AT zt(RfWTmy@z*5Vh>^(f?KgyI%<$Yfax*0&k)?XM{+kSU+4rnAS2?FXUC*9tF-+=Z3D z8o@&96xgYFpv*`(y45~`!(=X6UJ?sdC-u-Lw3&Jx^_ZIS6Tze`$)MDz;$pgVW$E;D#rz zbK&({-(jlB4$U7hqn=MZ2s|C28fj&)j;j}T^2k9i2P4+yrNCGPjnX*63ASRx)XAeE zm=^4>=Ndf&@PgESTFyA_Zx4qg!*I-p4wyCCC{ZgmJETY}=mtz5F+h`h z!pPWn4}xqYP&)qu%yA$k-YbbKuC9Oxg{!bOSOb&w*)jW|6gBU42<(LSz<^OI)bFIj z3WWk-70!d_S4*gzW3{kW3Nd28FWTkLfTpM`tWV#GtG+5>iu*ylmUIS=2DO6a!8+LB zAcOU1o8jrRO=zpef$R2bV&5|bJY?^NXG8fxA{TPv;FV^xzGF1DWY`277*yM$8_$C zkQ+e5vel|k`{4|Pjq74_^jGTgyc8xB9R>Z5w&>n0hP^C+ue*fskg6Z4WmM4n3xz)d zf-sK94I9etQH}C?XsTTcl6?grBN~ER-A}<(*cvQ2yA@Ma9?&Y?KY-?K8{D@q6ekyM z!J~8CR7~y$9Brt9wmqKIWf5&u{jmkzO%Ed09>(-NHW+pAGHem^#kS}A_;9g>>iK*d zd|upzz3YnMsh2&<)k}i&XDKv}7YC|;oI3VZ5{x7LV8xYPU=#cR#GIHh^};&j>5+$p zrcmsj9)hEF!I=5&Fy0^1!#2_9@J2rZ$KqzG%bsTN_9!Ffh4Nx-OA^(xtcx4!J7_MK zSnw>6HL4n2+N87#qUq3#jQNbLpp9aJ@JStU0$LLuNQGB^*Y9%cO^2 zpo*tvkK(UHJ4$5dY3Mwr1#R8h&{oZlHK(1y;=Kz>-th)ZI8R z=MaO_Sv?f}q6xJzD+!|331T7BezaEo0VjqRA!AVk&%I8BsQ!95Crs9B?4jGHVu3~L+;=7i?H_SMk@DtFdB{~K<*GdKKac9HdY*{ z$;X9(Ud`aTUk|U|+K+T=vtfz!fmE0bQT>G%-q-#J*SZfQWY9cTH@*28_}CXRp8y@$bQh)U?w>??GE`5gw9zt=X1D$dWrIr9y|AzBFG%z% z;Tr)V^!Zs2k4Me}Z%qr(y16hc^(93XLgQrxGGzbjQYhoUFGHB|U;B)#NdA}K$f9Q# z-?vV-|Ezv;9HV26reoHRW)PmFV>ORv5bTI%)}Ey2O^Rle(T`zwo@5a2i)Pm9h~W;O zTp__4!)mG@yQ*N4QO-Pu-LWH9uyvA2H7SPES3gd4Y?4{KFNQm;BTjsAl0~02mN!8^ zUQ+latGRjXs=SVP8SR&Bc1f}PmHNjOoL{m#_r(e}cN|v!HGu8^cSv1$E8-LaS4;eB!93mu8D_>w1@HC}?*Ajwep6>pMxyrf`flBxEq zm6=KLQZfd~md>wM74*f+XmuvrhQHz~XFV=wYLH@I@QT0M{J4T+XNqI%D}jcj<4V2; zsjg$M1Y7%#tAur?dMv&Y>R?S!O)yCF5`Hb*XP%&z*O}(4{aR!sDM6#sAU(kOwdh!1 zf>v{9dQkZ5)sw7=+PwxDp#`tSX3P_HA9rSiwZ2|6pOi?OHaHPE_F8jElOExz( z%+4-&BPC~%Z0XpQo!9zCS~WS@+Sf3rXzY!Qc7L*MSXWNz;u~3gwiLSr!`w5%Q*!1O zDfW3?xs}>e@^;B74wZ%{Yn-PPocmK8o4ZcdhfgVbvZXrr8s;?>Oey(Uq`E%t%4=?& zQVvN@b)PoOZy%dd3GYw!Sm?^Xu{gCMnk~(f*{GmPcv>~dBF#(iRza`!^v2BOG;bNB z!U5-LwSxXMU#(k(!{O8F!rw?uZ#93HGccQ{ywM3TKSv zEVEJ^yUz%=&6udBWTp8UpA~&FW2!xnl@Zo`R(xs3OrJeFGr_n*QskYvxn*{CUU!9z z&N~aclR4aUE?Pr-&^tiiHtL>dlNJ>u0 zv~iX0lXte^139G&-BqykZgVtyZaK3_wV}v+yClopGlD(UraJGpWTxa+$e5hBba`)I zFpyiR)pOo9;{Dcg_LJ47CN=hj?;WZwPu4j0)Ht@i-`0?Fvewt+g6os_j;#YH>%)33 zcr3kl>R``nNHD4O68YfVXPMWO*Hi1O^TA~#CGT>jNnL=;2iLKIyyoVfx}b;;Zj#7lMfz?1Nk==dM+Ma`mlqJqo9M?v>{ex z)|1t$pi8i~Awg$$CvR#&w~T3Hip#8*@L)l&R&QfQ#Oy8!j>0}u)28ggS#LS3!U4zL zro6UUAJx>tAz#x=MNejZwFe7_!+I}~Hsx-8j-rtS)5~W>KKhwk72VJ4yfk7yPcXaoO5{^`pH=CXyxZ5Nbv{Lmq?UfGG`s%Z5Z29r+8w+ipjvSj$E&DZXc60g3r=yF5Wq%fK-~6-m=@=bnIUS362ZQK* zG^=$vgHT@wv+jHhZ(2E{ta&H9>wK*6P&xCazE1AQ`8WyA)2wFZU8{=b0z=VZII zGyGNNJqoU$Q=Es+2)6X~s6>8F_2fJ&eA~QNt>|-_pY>VM(Y{`-_Rr}dX=laWn%~xa z`Z*(f=&bmUzT2?;`9w5lg#?R5pP}fN%p~gyNumBeQ{69FnQ0YLvKIZ8u3xeXhALz> z_4nIGe#t54JSS&nF<@WxCAZr8oPtyTfMffYlMQL-ly+MTx<37q*E)1g{aU&>R0Ti!?;c(LT1LlJtEj;G;l8?^4nPnuJh(*miM!ZzE#NCoVRouxS!Yl z?VM`*dF$Pl4~m|CtJJ=8-uBSIgVN=1Rr*{tc8Qh`&xn4nHn*v<&mVYLsr&uBU3!f} zmF1%v*Y7pXcWNA41|HQ%e!t+!b;0?z<>RKJ@3nq57hFdN9yhmtuM0`P;QrQfwEgM# z`tUm!Jbny}-dO&AF`BE^lf`PROLUdAoXLQ}z=THj5B zPlh8GE|qiD`I%WgyO=Df$ES6Fw2h?KAE>f={@(RR``DfOu$IB+bCEx;PI6rgzil<~ zrRc{ss~MY%k)wkX3++Fy&!=CEdTaG!`RR`vi+3&_{W19B&+?C(bleTmEY_0@s~0<1 zZ5v{Rh9;Ty7CU(}8scTGU$VO`b_owRBy1Xb$$fb7mIQZWl9~0ZRmF?la<+{rPD8H* zuP*keW;CYlwtg-8Y_V5+xH03<&};FZi?{W;n=%uv-$<@r>NB@(%FZ8pBcr#}ZuI&(r6E7tOGTqY(^^-T?u2AqDtT-DR`=P` zaQN`0(jP-_;pfubXzt79EH*QStCvTTY%iY?x-(;{w|p-%<8p&KA+KY?XAtH$Y(z%7KdAI{J8V!=+B=o=y+N?SZwEGSO1!1 z-Q3zGG#qkEgqu#LeVSfOfb~B+^VoI;2Zikj-Fd+0f9zA6E1(yQEu@zjxk{obSa@l+yAZC?coyd~h0 zxt*H+q=RdMMNvELG^KON98a-NLWTudIt<~Wd3@l+v>JPOR%wAN&$FSl&rOKKOgLOH z077|)y2XEi&OZaZJ-r7&xkC6SE#LxI#|SA0VfvU!LyKFU=zCnsUxzep23e^ zwE>XTTm=y))8J-&3tXqIB1=2#FvNEw*4SI3oV^d_7q}aBtejzA=spZglEagU`>|U= z54*K)LMI0|_I9YFCAAE~yWhc|ZJY6As3BHZGos1%Xt-_mN{>JCBB-(zQ@*o}u-Li< zExrWereGRAjx<7cc0PRhY8P(lcEG25dTEy>BX!rC4#0)hGr+n=09DjYv3<4$v@D{~ z%U>9#uH1p&-8HP{VWPXsmTT@Z6v5dU~^A${du+6}8|YGY&w zno^J8G?NS_WS7C)_lBsqOB9;5a=;J=LEzp?$XslOP9srdyl8}4iDb)*X&^jLjl|zt zg&?`Z9IOA-LeU0i^!5pZI&Lp4xXT3H-dgxAk`JwV#DR&`2U%84!OX6GsIa^S#rG+q z!4qlxo^%)=-uMB+&utJ}RIxb75`P_8hj&j0;jLgxytcs}+3&gG-y{|=-K2xM0-|`R z{5LTC)y9|@A1pY>i-*krg8uLv@EjGuvBJIh-C+$lL{S*OQU$lY)xnjYk6>?K3#eSs zfnANC;mLUlEhaYMk>bMjO{CE_HuR@C1QjgS>qJvPz$wdb1saWnx#Gb*aB)p)=*fZj2|nq zsl{W9&?_X35>m^Q?#;(=#@QNuCZ=e9yvj)T-jll1ri=!%3sm1~CfIK&iKL+b-zU<+ zpMEp=yfXk=@fVuz=TM9t>w+zG2k}p;1McRZ19=%1SWTA4W(@h^{Jyu;Ei(NZR9lJR ztcP*bZj9!+O9n2;+hMl#Zj@5H4(n2!P}3m>(snyytPwpPUm*^vYNtU}R0zZ@LvW67 z4cxQP!AIwxK*hzaxRyl(KV6K(GastKeCaIp{ZS1}Y;C2Do*xAH24Cv1P%B6qDkIxt zQ3yU?NmcKhhsBa%$lodr(N?8keyRfWf*s+{s&rV~PJ=beKWH#<1X~OJv4!;u?2~y88C%GflC7***?Ji) zk6(mq6D&A$$rszVYvZo3R@CQLQ3#Ju!J#x6^s^U5@#kU4*Jy>z8JfsJX?@Q#ds+|&w6>>AQ=Z7e`{fP3*z=$b%4KY@OKH>PNc&D`_sj6 z$=(jW+Bo6+<%^JizaP9;GN78W6Q+rBV8Rt%w67e2x1UrXcGwQLH5|hZZgv=qmBJlY zj-a?$A#GS?nTmR6gqhhoSijj4El8{Qo{`Lz_ZC$U`w`s_9o00H5hv_&N6x-qM@mx*l00u7@N@N)$(p0757>Anccjb9b_ zGCrXmpQ52#oH*uNy@b(&vcQq?465l_sr*OD)aLX<=pd(o6TurXM#2?~Gd)1;(=2S9 zw!zh>D=A+V5sEj&69qSZ2X^I^cxg@^zm7bnlwWM2)@+x@bcIg9*g`mTOAf!1<+p>+ zyn$ogW^5UJ3C!Y_xZuAMS?=}lR_C!=R(jl zg%^IAKwEPpo^|M^G%QppJ-4z0u#7qmXT_^% zF#;@baZnidC!|9lmm{{k*as^*^PtL`oR9bAFxk%-1ELn;ZPW%}+8B&R$3yXs{5R?< zrxrG~@5XE?9z4A91Z63A7+3!GN4q@{5bXqv%=z32V(?xaAXyQJcyWx+QV$M;R83KI3PPR+VP|i*?*ukieztS|J zr1uGwbR4Gc+F4=FqzYx4Y>vP4ZSeBlofwv3k0ZYg@k<~*`B#D)Q1VtAYm-JGrPBas z8k|wUYYplOvS6xL06v%0pu)M8agnpQ1}50HlnmP6)xdd% z8{6$y!P_SsxPHh8I6f(3Zc#Qk(U-w#mI$CDTj|mTHsM-f33Lu+!$)5(fM|6x3EEO_gS7tMusT0*(o6m;#c=f?I z>^k_8Wsa(WKr}TE!UG{z`29^3N;x;ehK+(~^k$B1?XUuYT`Z`1mD1B43d)~p~*QX1A-z341fV6hzS+5 zB48L$L_lBa7S&OWE^y>+X)s;IuVD~XhD zO=PO`2GOm0d74H0*@jcfBqb%F5)m76GVr2jqYkj36FgF?h-UlCH0kJuA#_WynQd^G z!*$e^vgK`k>Bs#?*^dgl+4eV`yyi!3>?|*tDletb&6+0O+8koJ6*2T7{s5aYYbYmo z{2`O>tY^Enont@g=5qDx>R54P5o;ajP4}1Jn6Xx#y!>ae8p}s)f2}xq%`W2-m9{e1 zutm&HB8XJ^_zF6p1Lsvax?FXZ>ET+(eBvXv%qgGSX=g@`tbvJ0ds5)Ww~SZW%nChn zsmJYJ?A698G;`(_?ndJvx_38&UCGL$;jPhh?20(MvbvnbHX5-=31b>IWGMA-x1!z0 zZn3$kNu-#4ojr3^p}lKQGf~&=>}I(G`Fg))yLCp=sEz`bIp7AfdMQDVQl03|#y<2q zcsHBZVhcNx6jy`7Ap&AoVEug)h3aV z&RQmadJ1svB4>YL9kr(W&&PL>^k$v$hPU~bcZOZVY@%Nu_I}&@)z{QxtFkO%2 z=g8BUhwUuqnJH`By@3S`yTo2Q%wrS2A4*+6TTx0^F)Pl`WA7*Gkn%%+TG3`mokte2 z$lkg%$ySF;l^#XBt8t9Ca{{}pYeS_^_1Ni|ax`tj0T%X59#a=wX@T8HRxmA_m9;5x zijJyuFaAF3U3-@e>Aj9M>~N(}4A<_PJ(!}#uViCQkFd60m)Jc?Nt)j?iB81!BbO>a z-jGIoMH{k`9d#6A>S2djJca^Y=&0pRPs<~hbvERX6+vPzDp>0)NoIIloPLUU#nbLZ zR9SY4Wjaozkw=V4`JFw}Y4RkVm>Bt$Po#q^l=k>9;!bUtM$PM2u?HKsvNYKm?po4L zwzEE(##;5HbEdlFn&3;J1-YyOpV66)``PV?Sdx=(=2CyY#)TMWu?BHq8jC4E=Tei^RF z#PYUs>1Jm*8Lu~Ns@fXf9+w(c9vDuR%e&Z+dB(JGOB~s{=5Pj^9LU^Efg%FWvFo>b z(9?Z95Jxee1WS3S?XK#}C zv%vg3+SkD-Z&@;vS{+4p{f5)A8B(mvqKMVq@}Y_zwe0zo$@F@fH+QYK44aS^Pm*_z zGxhVfWZqWJ7VQ3kyZXx#_Ij%xod_F2cizb}hdb^h5-ULxrwr(}mk)*YPX>PO$HQ@ND-#Lk@XXnwe+O=%S^TW(?Pfy-&y6c(Av@Y&sC!=bQ zXnG|0fd%txS(BJXUFznDoehCC5;L- zr{81owOG!6)+{le%&VkGUEPl^ZtY?*`}LWcpFKsCXp-)E0aISr$#{n($z!Aj8_}4; z9v^n&-acW?wYKP!qOp(bSZ@Qy)yoc|`gu#(bd_LQ-zrU=oAgMm^*v|O zH<(1?o!QpGM>u^WeKJ$p#uAS%WXWlIv~sNhh0KVcLn;-lnjm&f35b`RoW}>I`SYauTbZ+io$(oqzbWMjU z9H&xR;w{#`j!#C-3s{Hmb{4+k5j!WUNKxXt%yS&RuDy7lyK{CBn-K3sJvL6{UHM&^ zWbBr)MvrSO6zAsO>ld@iB^U;-k&dsgz1jQZIoyN(x7gJQQ>k^(URHKt47Hz&pjB%| zkl6=Y(pZgi>#Z)99O}j@zdOLb)3jsKbJ{tV8!gPeGLbIKna^$K9b`)3D%2tx#YD<( zvPm*IRIE0dS(KWSzSIaZmJp{0UGJFKW>dCtgERBn8A6ZWKICPK__K*x``A#oD0aVo z57S*Zl_s}kGjB@~dY$|$)1N$^sg}*-_0z6lS^aBRP3~YaxO$pNXlheg(_k`QnN0Ub z#c=KeCNS|Ol5GB+GUmOdneEZT^|h!9_g>SL&Q0~9Yn!ICX{yJV=>bIy&p6C{`pxEC z*JV;x71}*Hf(kFHljE5I6kaof4N3WqtqGE+(J!vDj+h)epMHnYgcYoG ziVwY=n?ZM$o6y+b(`nr^MrjgJlodaWrth?1b(}5P{iIK7eVW+w!RvVSYnHLcdvVNE zEay!eWlt_y-)*V+`2Yje9ZN zOpI<0RwV~7J>JXOF4k~)8{6AFjh1w7=Bftbc(C#wQ+s)e>vgb>)eVfKdxzYapSuGy z9&bimz;bS#S0j6VZaP~rcQ~0oiN#QyxlGE-k6By~q4Eb~=#I}7*5WJ1-L(IW#pozf z*8~Zgfv>c)Jw&N*Nk8iSNR#YLajrgYNmHUlxfLxP?D^5r7;G%3FBdN5y$G4f zk}lM;+u>@IIr}*y^OH=?W(uvYj-{^Nfua^O|yTS#hT^ISj^FKX5BA| z_G#>AQbW_}yZVc~`mM{kdDHah@S-$U^}&+$-1U(4vd8~Oi>`3CHhdb5VVRwUcRBry zGEDb?tYz#CGuoV1z|2k7Guw)4roSSXYU>`e1VRE zX2JC#y?O2I?WFVUM1(GPJYyL%{-DPVYU@EO(oNX+iwDyU4>LCATmy?Pdd#LZK4Ebk z{!}t8mX_oRsGs*5rZ=pB?a51_8H2Lf-HL(aGiVj7$ths(RMxPh(H3;Z{16k9ZD;pt zE7|%qCla}2NHeO>u-n>W>9;jGBs1*+dq38lHgx!sTz^sG8h3Lyt47dvZ4r9W>PaSV z#!{~5J!b8wO$|%?(gr7E+K*v29VVvi_Lc|ir%g+_WAA!X;lV?!Fkn9Ox-QLn%oL%_ zRVUf}O#v(&L;D2lIvDSJOBSbglD++{jXjcl&8|ikFm34&CNOZNhIf5ffsYUCQ?E-3 zCcZfCyk_%@hSKbb187F_a~4uEp29B=rv1HwXiC6H;(LkGtF4aI*QPxO=9onjv(vZeP%$~FT zyvd~HSj4!;qey>U7(J>^rJ(n!v|3i2QeRZ?`sa#RRmCZj#_$hJUSkE%H6xB{?$mx>IqIdR)IWj zgj0(6TK4j_4E6KzAaPEbSz#)~pnmJPL3LT|N}o{ny^EveuRcEX!YG7l2CicZc5des zo}OX*Dh((j+?KkY45x$Vg2>J;fm@uPL7TL9a`x*)DL1>3jqiHHvM@z>M%+-^w%mgp zcBXS)PVMYyTLa5KHjdp}5yXyQc+Fd}V0QP(K(=?~2(~D_Cly&*lkH)SsfVOeU#3e5 z)9h&=r$P17XITWMY4Cz&smIYIreJG8FrUp>r@h%s$J*eOztp6i*wwn zr9;Tra08czq0X1{hEbiI2Dv)i;S|jCxVQHdsd?fZw%*H`)koug)xLe4N1P*F`t2B3 zeAAkl=Ll#^eIiY29Zr%F?QF@-=}ec`hrRH!=LxRGQ-aSdrg`6-gg*xUhdW~n}S;sam`<}&nJ!jqr^=SU>X*56ChwkxZNlV_C3sa1y$x6R*^G>Fa zRQ(F3>4YhKeVf_y{8nyFjx|a5U(6cArqRZ2hHS+|8PeXVz*4UEr+6z#T6;Bw-u~)E zeHzM{xlaN)x{oJmvp720XAWo3?89C7DV1LQGK%Ujz3sOR(qy!C2x-NPr^SbC=xsFW z?Wak!KAfZ!htSjOV<`wzh~GN7(1Xr2YS{9gofNsxtC}N8>X>fRaB?fRKFx;=Kgf~Z z;>T>qH9OKjd59%kp2>I*AF`zvUookaOnN_m95e5IhU@iv3(sIr7c2kWh;Fy&aqT6M zRJdgdDcvY#!+tO1{k%t)bS#wVrp_|%^j(g2eo&)bS&5_;-Jfo3-^Sc^?WpO14L#mE zmFlOTVD{(xbD4oAEMVO=rm#qlv^QMjCTg1E8d;L$hWDV8m3>*tCEQDse8xI&hmh{W za5}sug6=9g)5^^mgh_a`JUobGP5W^JD@BRF>K1ERJCw_l+Q-zUPohn2h)K zq3l=J+0d~yOLxR3n$|c9zH%$3lDc9DwH+2j&A?^5Q%&*FuBu%ceQwu= z)#22Zp+Hfi(x@V&lxNCQrmZCvtjj$UKV4u!2TKh}Z;f1k{>_Y7w4gCuC` zQ(d}Yo=bVMPP90#l|B4Dmd0KGg@ra4(d$A(dV?uF6DLX1-8|f$-y1{oH)S#L^I|mr zdK5jn5=qq}A#Xe^fSGbmcAjq&hRf`ISu$% z!M?yr=v#nkuvY>N$e9ejfgD}vJ%O|E?*L)2Uw}=KGaGyzFhE`x^hdx!`1cSqge?l* z4;}&J!(NBHBE+AATy7WQ|5D|~bC1K=^x6Xk_~ zPlFeL8RBN&_a&p z0Z1dR54=Z?7NCUKF!)!%k?=Rc?v1?Ju(txmfF589eLVOLcrbED!u|p906L*P0lvd> z^WevVhr%xbPlm50tRL6_`#7{z*kwQ)axTG_2IS#e03%>$LO%fB0e%E+51ig zTf}Zb-wMqeYzPDay@8=XHgFg@*ML!gJD>vXJ@VzTye~K%S`>H#E%ey|823p+rl0mqOt0{#hvLV=~g_s}b$Gq43>?$DkKb=bw=iQvOv zQN+t&uY)ZQIKp3s<&40IU@K^CuqT0Ckyi^|1brg3PVg-FC9p3c_6G2PtpR;1@B*>b z&<2B@p`U}j8hS2#ORy!htI)2(H-ViEv;Z3sn+I(rFbMH3Xt!ZY0{sxX0A2wO1Ums^ zk;8xqz)1K5fD5oUgN=ak$W;`^P)-c|h2U*~J+#3e%PJhRInTJpCdN| zkc969mH_h*pN-fn_yYJG{1o`nz)Wb~KriSe@JkST3uwX42ec7O01pHI3Z0NQ2p9+a z1V0#92^1sV2YC~~vfv$vw?of^eGv8x=xMNr050&W;ZK9@4@AONfmR256_%L`egkb6 zG<&c)up8I|KOd|PNFgU0xj%z1g5QEOz)64(um(5^?E-Sgf&GBnus48%QH~Yj0{AiD zcCa?IEr2v^J+L0~8i9U5IQ*x`lLT(T7X?co&mF7@EJeH&`mexczyeqYZ7;A7wjFXE zkv9zfZ{VHaR_F_${|@#6egFy(^8pkQ9|LZI76F*SS4Qr4(3-%pKm%Zpm@_napbJO_ zOrVWLY!a5y2Peay35oP;(4{t#d$a`k~D(BcrA4$T$309r1zeLys9L%A+@a)8S78l;J-DKY*qO41^y9-Ur0P)&%zy0`_y{odQO|J_J4v zy#w`l!qx{K!k>tk2<&sP$AJyNH=*f*#ex37FVMB1^%MHwD)>C`D{voZ0pM`p8j7+7 zJb*C(AIL%8Z{VefcOZ8jzyr>}F9pv9<{)PSuoiHFz8ef#L>%^M*gb#&5I`a)@W#P@ z4Oc`~0y+e;F_=m4OrZ_SE)-k1!nd&Xd;fq6CRHD}}nl&(P=b2f9k{acTR1G`$ z_Jad2rSuq*^I%tz(v+@S$A?ar&}dbgDZjf=y-2New#GceA6i9vk9j4nRpOx0sI}NT ztWCQ-e0PyfR{RmiVQZ#{Iq6mv{~$7aLwRa1z0K9TdyUvu_o~;(U9Dn#%{`}&IF72l zS|Fug|6q4-gG29M^)_tsS1TTPRMtyIqeXRwi*ftNqb??=c;c?6Ke*Vqnq4SP=`;Fr zv`x$CtLb;<_qj1W&2`M}VjDM&yCVgsESzRfce8v{bJWf1*+KE~yjLBsCAl}3()zKV zPFH@x`|u zk9)~DrILIPyLr)FN9|NIemrBZ=Gwp1!Kc{P(^01+t-sUo&3pShkE-wL?_$(uTBavB z-R$T(`r1sdaTX8vdYSW7_KX|P2EQ2KK0s3YoV%?;#5s?l>Pr@TILK618P$tB;c;kC+wu_EWn1q<^f0>bl3UG0{8FA2yD@Nf%Vx?E>P!oB`c zt8?C}M6dlxtuy-iBHtA;ZlMv!VqR7*y&%}U`JS?3kKrXf6}W9JfpPDZyXM60Xi{7m zzvqna$^>8A!ls0}yNWXt541H7@;UU=#^$6($*g(FR`X3&Q;y}2ot3h1aM4vk+wh{M zVEIwU6_mscFJARI#SQD7?&~ibB=&=={WZ_!K2d{>hX)k}WfsTm5Arrnv3JgDth{-x z{|zI_Nj-1%lq$^*>6tRuu6@4!kiK`9o44omF)g~Da(u(4>$y)|LX;=V?Hx9E@|}lo zuZz5Jm93ca`i$Su$qS?fH>NH(?J+3*z1h-=Y5kv=OXS^rvuS=tZ{o~2in=!njiuzC82l_iSbHNRKRAJ*!y$XmQ`ZCkPF zB!5kr*vr|y^UQCTuUlXhR@84{YR%~2g;77f*IZQJmTbFdhqi{6wN0d5c!}MR<$X#W z`#)G*>TJ9jzg}RbGQDhkX=cz!_tBDz=gb>7e97WbE{+jP1pW`tg)SMBt=*URgVqnT z)x&zuOVN*TJUJmyL%TR`RIgpvGJ7`I%Nn?4?>)M--J(%^ndz>J&N4x{7OinBr}n(( zv-iEgx_sGe3!UW=D*|MEqNgZmj}ZNH~-o* zx^l!po#;&EhJtIA@qwFVf*e;_M~^MctaeSAH6&N1N=*CbHB}`e+oNNYeyuvLHz(tK z^*WKoFRQ1|QIS7osku{Z^THCRT)V(B&&~ydPK?Z*r)WCFJ*NNo@UVv)Jlwq&PR*{F z851xmUEC{T^v;A z)XIy@X?s;vp|dD(3uof3;Wy>|+A5ujn-a6bi??zik2asLoZDfwJw_@-Ve_=LkG6E4 z>pGXTtS;d-NzR$6cp@zav%NS(AVLoLtR}OUbSJyDT*qX6$O~Tn`*bO@i!_xQe+?E-(Rk?A-_7+3&H=~OOPZ6=W|KjMA z(`$=L@0a-WY(APlU{~{i<63t4iXKn&XI9RSc0Tsqqc+2)=6n9_mKRmemMRt{=&nAd zl=k!7zRHg?6Iuo)jc9Mlt`*y?rD}C~7ynT92Q}WnsV*|D>IdvgBxOsqUg?#-Rfuf5 zw|bqZ^}+?KRNC$?{cxSh&AwIB7Vl`&InB~eDPQTr+>5K+WY4rGSrw|tiWv_xm}9(A z-R`uYU9YIYc+klGPmDzkwjND7EO*3Zvz$z`OSPKO1kr6BD*`T*R)_iLUz=yuA=7t} z^0C;SC*_;mO+q%BUOuHWy;If5wdZ{6J4V!qI2&W*5Hrlt@IG2Mqoi=uQtHPLjG7OVd?!pm5#^AsHm#M{|_HF z^S^n@%YSc@zRrK`JC^_Qhv}(8Q<+jge{^iJ=J*5E3)u4ZH26 zWSB-{49&Z}h;DCc5S))i26vYq@u}p$78Ds15g#7>SM6iVg41)7g*9qqB*qAm!ap`5 zBOw{buTO!bu-FJey0CEx5$U1fq3NMWpR5s)i1e7mDB-d`-gcwLXiWX0F@J4^##hg9 z5*GHCCVea6tM;Md2;oDZ(sR4}=`T(EBR1M}LSkCb-Sb%|(vxHmoFI9d%g#EqBzl6T3GBQ3XG~LAb@0ETL z_)pdT@uleBYyC3v|GLt@o)EuL_umt~Y6AN7TWkGU{!i=t>1_7*YQG44Rox$Z)8G5~ zkH|Oov+k|j{c#kYK$B9tKPB-oVX2|1Il>P~a!zP!YEqW4R6N})EhZ^ZsE*b*)Hl+= z-l6fS6<@@R^iA~5G^UCu%kVut$NO1+Jaa17$QlS=*-`dZHPMdUc8hWeSsImRYMIVL(qhb4r^JH}*1IVPAzhxz#kb27sH zveR5__%Q+#&uBrS2Q+@xCqGkQ<2c&UcATlJtsu?O#wj=4-Zx8FmaD%}xG*Qc*di_5 z&nUy+_+$Nn{-5j32=FtC7v_c~`lf~12v zOt_y}GRlt&^l}u|jdmJkMfh46M)+HIm+P2d5fhr=8yjxZ9ZwJR_l!mvh5=q^vqM@; zpr2W6n6YPaSc0ItJYQc=dl&Dhn8>f%F33T?T{`;kk9P560{sKy!xB9s!|mfyuA3p& zCF^6qeZ3CLsE>e579sA0*Cm>aEJITtm|m3Yg~Z8 zXMA_v-T{Vw#%8(xAN$~M>-e#K$(dmZANvgJXzQ40WZ_~X+$X+R9|x!8@C4r+Y^S(? zT#v7FzS;hGOb-Y!{Im}lh51{XeYwrBuNk_)_M9PJs#W@nQC!kzomT>7TY$bbztjm+k-G zwaq^;(cjoJI@B*)xD9eb{jGEVY1;<;Q|>=*p0gopBE;hTf-jc+J? z>ps4_=Z-Ki7z1ydv+%_5aiLG=2cgdwE^7U`|M>Q)I0*^!UHQ)LFomZN56ABF5l$XL zr~6&_W-E-@+ZhKMS;PhS#it36t%8L3EJ2oa5|%XRPV^GKTStBJt@+vAYIl3P+t=+1 z&r5v%7oU}G)i4uFb=Pm>@zwI(XTmR*cj{i=*dil5-U9va8IRKVHfWuYKfcECv%gxO z?uEWs9~TfWgFFZC2881#m ud1!KIYDz{#`Y1ti@~G}>5VcPi8EV~EA1d+q^6#rZk?r_W*PZWa^nU=H2Ysmk literal 0 HcmV?d00001 diff --git a/go/cmd/server/logging/feature_repo/example.py b/go/cmd/server/logging/feature_repo/example.py new file mode 100644 index 00000000000..aa933a11dfd --- /dev/null +++ b/go/cmd/server/logging/feature_repo/example.py @@ -0,0 +1,40 @@ +# This is an example feature definition file + +from datetime import timedelta + +from feast import Entity, FeatureView, Field, FileSource, Float32, Int64, ValueType, FeatureService + +# Read data from parquet files. Parquet is convenient for local development mode. For +# production, you can use your favorite DWH, such as BigQuery. See Feast documentation +# for more info. +driver_hourly_stats = FileSource( + path="/Users/kevinzhang/tecton-ai/offline_store/feast/go/cmd/server/logging/feature_repo/data/driver_stats.parquet", + timestamp_field="event_timestamp", + created_timestamp_column="created", +) + +# Define an entity for the driver. You can think of entity as a primary key used to +# fetch features. +driver = Entity(name="driver_id", value_type=ValueType.INT64, description="driver id",) + +# Our parquet files contain sample data that includes a driver_id column, timestamps and +# three feature column. Here we define a Feature View that will allow us to serve this +# data to our model online. +driver_hourly_stats_view = FeatureView( + name="driver_hourly_stats", + entities=["driver_id"], + ttl=timedelta(days=1), + schema=[ + Field(name="conv_rate", dtype=Float32), + Field(name="acc_rate", dtype=Float32), + Field(name="avg_daily_trips", dtype=Int64), + ], + online=True, + batch_source=driver_hourly_stats, + tags={}, +) + +driver_stats_fs = FeatureService( + name="test_service", + features=[driver_hourly_stats_view] +) diff --git a/go/cmd/server/logging/feature_repo/feature_store.yaml b/go/cmd/server/logging/feature_repo/feature_store.yaml new file mode 100644 index 00000000000..3b48f432875 --- /dev/null +++ b/go/cmd/server/logging/feature_repo/feature_store.yaml @@ -0,0 +1,5 @@ +project: feature_repo +registry: data/registry.db +provider: local +online_store: + path: data/online_store.db \ No newline at end of file diff --git a/go/cmd/server/logging/logging_test.go b/go/cmd/server/logging/logging_test.go index c7b7fdf40ec..9181922a47f 100644 --- a/go/cmd/server/logging/logging_test.go +++ b/go/cmd/server/logging/logging_test.go @@ -28,8 +28,6 @@ func TestLoggingChannelTimeout(t *testing.T) { EventTimestamps: []*timestamppb.Timestamp{ts, ts}, } loggingService.EmitLog(&newLog) - // Wait for memory buffer flush - newTs := timestamppb.New(time.Now()) newLog2 := Log{ diff --git a/go/cmd/server/server_test.go b/go/cmd/server/server_test.go index 11fed1360ef..5fe4e732ae3 100644 --- a/go/cmd/server/server_test.go +++ b/go/cmd/server/server_test.go @@ -19,13 +19,11 @@ import ( "github.com/feast-dev/feast/go/cmd/server/logging" "github.com/feast-dev/feast/go/internal/feast" "github.com/feast-dev/feast/go/internal/test" - "github.com/feast-dev/feast/go/protos/feast/core" "github.com/feast-dev/feast/go/protos/feast/serving" "github.com/feast-dev/feast/go/protos/feast/types" "github.com/stretchr/testify/assert" "google.golang.org/grpc" "google.golang.org/grpc/test/bufconn" - "google.golang.org/protobuf/types/known/timestamppb" ) // Return absolute path to the test_repo directory regardless of the working directory @@ -71,7 +69,6 @@ func getClient(ctx context.Context, offlineStoreType string, basePath string, en if err != nil { panic(err) } - CreateFeatureServiceInFeatureStore(fs) loggingService, err := logging.NewLoggingService(fs, 1000, "test_service", enableLogging) if err != nil { panic(err) @@ -103,7 +100,7 @@ func TestGetFeastServingInfo(t *testing.T) { ctx := context.Background() // Pregenerated using `feast init`. dir := "." - err := test.SetupFeatureRepo(dir) + err := test.SetupFeatureCleanRepo(dir) assert.Nil(t, err) defer test.CleanUpRepo(dir) client, closer := getClient(ctx, "", dir, false) @@ -117,7 +114,7 @@ func TestGetOnlineFeaturesSqlite(t *testing.T) { ctx := context.Background() // Pregenerated using `feast init`. dir := "." - err := test.SetupFeatureRepo(dir) + err := test.SetupFeatureCleanRepo(dir) assert.Nil(t, err) defer test.CleanUpRepo(dir) client, closer := getClient(ctx, "", dir, false) @@ -174,8 +171,8 @@ func TestGetOnlineFeaturesSqlite(t *testing.T) { func TestGetOnlineFeaturesSqliteWithLogging(t *testing.T) { ctx := context.Background() // Pregenerated using `feast init`. - dir := "." - err := test.SetupFeatureRepo(dir) + dir := "./logging/" + err := test.SetupFeatureRepoFromInitializedRepo(dir) assert.Nil(t, err) defer test.CleanUpRepo(dir) client, closer := getClient(ctx, "file", dir, true) @@ -275,44 +272,5 @@ func GetExpectedLogRows(featureNames []string, results []*serving.GetOnlineFeatu Val: valArray, } } - return featureValueLogRows, featureStatusLogRows, eventTimestampLogRows } - -func CreateFeatureServiceInFeatureStore(fs *feast.FeatureStore) { - f1 := core.FeatureSpecV2{ - Name: "acc_rate", - ValueType: types.ValueType_FLOAT, - } - f2 := core.FeatureSpecV2{ - Name: "conv_rate", - ValueType: types.ValueType_FLOAT, - } - f3 := core.FeatureSpecV2{ - Name: "avg_daily_trips", - ValueType: types.ValueType_INT64, - } - projection1 := core.FeatureViewProjection{ - FeatureViewName: "driver_hourly_stats", - FeatureViewNameAlias: "", - FeatureColumns: []*core.FeatureSpecV2{&f1, &f2, &f3}, - JoinKeyMap: map[string]string{}, - } - featureServiceSpec := &core.FeatureServiceSpec{ - Name: "test_service", - Project: "feature_repo", - Features: []*core.FeatureViewProjection{&projection1}, - } - ts := timestamppb.New(time.Now()) - featureServiceMeta := &core.FeatureServiceMeta{ - CreatedTimestamp: ts, - LastUpdatedTimestamp: ts, - } - featureService := &core.FeatureService{ - Spec: featureServiceSpec, - Meta: featureServiceMeta, - } - registry := fs.Registry() - - registry.CacheFeatureService(featureService) -} diff --git a/go/internal/test/go_integration_test_utils.go b/go/internal/test/go_integration_test_utils.go index b0c0205d709..cee8d5f56b3 100644 --- a/go/internal/test/go_integration_test_utils.go +++ b/go/internal/test/go_integration_test_utils.go @@ -85,7 +85,7 @@ func GetLatestFeatures(Rows []*Row, entities map[int64]bool) map[int64]*Row { return correctFeatureRows } -func SetupFeatureRepo(basePath string) error { +func SetupFeatureCleanRepo(basePath string) error { cmd := exec.Command("feast", "init", "feature_repo") path, err := filepath.Abs(basePath) cmd.Env = os.Environ() @@ -124,6 +124,34 @@ func SetupFeatureRepo(basePath string) error { return nil } +func SetupFeatureRepoFromInitializedRepo(basePath string) error { + path, err := filepath.Abs(basePath) + applyCommand := exec.Command("feast", "apply") + applyCommand.Env = os.Environ() + feature_repo_path, err := filepath.Abs(filepath.Join(path, "feature_repo")) + if err != nil { + return err + } + applyCommand.Dir = feature_repo_path + err = applyCommand.Run() + if err != nil { + return err + } + t := time.Now() + + formattedTime := fmt.Sprintf("%d-%02d-%02dT%02d:%02d:%02d", + t.Year(), t.Month(), t.Day(), + t.Hour(), t.Minute(), t.Second()) + materializeCommand := exec.Command("feast", "materialize-incremental", formattedTime) + materializeCommand.Env = os.Environ() + materializeCommand.Dir = feature_repo_path + err = materializeCommand.Run() + if err != nil { + return err + } + return nil +} + func CleanUpRepo(basePath string) error { feature_repo_path, err := filepath.Abs(filepath.Join(basePath, "feature_repo")) if err != nil { From e0a4ec67eea3656325a50e93e18fab54d1f9ff2b Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Tue, 12 Apr 2022 18:46:56 -0700 Subject: [PATCH 83/97] Fix Signed-off-by: Kevin Zhang --- .gitignore | 3 ++- go/cmd/server/server_test.go | 15 +++++++-------- go/internal/test/go_integration_test_utils.go | 5 +++++ 3 files changed, 14 insertions(+), 9 deletions(-) diff --git a/.gitignore b/.gitignore index b285234c731..477d2411fc2 100644 --- a/.gitignore +++ b/.gitignore @@ -105,7 +105,8 @@ coverage.xml .hypothesis/ .pytest_cache/ infra/scripts/*.conf -go/internal/test/feature_repo +go/cmd/server/logging/feature_repo/data/online_store.db +go/cmd/server/logging/feature_repo/data/registry.db # Translations *.mo diff --git a/go/cmd/server/server_test.go b/go/cmd/server/server_test.go index 5fe4e732ae3..fd0298da8a4 100644 --- a/go/cmd/server/server_test.go +++ b/go/cmd/server/server_test.go @@ -54,7 +54,7 @@ func getClient(ctx context.Context, offlineStoreType string, basePath string, en if config.OfflineStore == nil { config.OfflineStore = map[string]interface{}{} } - absPath, err := filepath.Abs(filepath.Join(".", "log.parquet")) + absPath, err := filepath.Abs(filepath.Join(getRepoPath(basePath), "log.parquet")) if err != nil { panic(err) } @@ -99,8 +99,8 @@ func getClient(ctx context.Context, offlineStoreType string, basePath string, en func TestGetFeastServingInfo(t *testing.T) { ctx := context.Background() // Pregenerated using `feast init`. - dir := "." - err := test.SetupFeatureCleanRepo(dir) + dir := "logging/" + err := test.SetupFeatureRepoFromInitializedRepo(dir) assert.Nil(t, err) defer test.CleanUpRepo(dir) client, closer := getClient(ctx, "", dir, false) @@ -113,8 +113,8 @@ func TestGetFeastServingInfo(t *testing.T) { func TestGetOnlineFeaturesSqlite(t *testing.T) { ctx := context.Background() // Pregenerated using `feast init`. - dir := "." - err := test.SetupFeatureCleanRepo(dir) + dir := "logging/" + err := test.SetupFeatureRepoFromInitializedRepo(dir) assert.Nil(t, err) defer test.CleanUpRepo(dir) client, closer := getClient(ctx, "", dir, false) @@ -171,10 +171,9 @@ func TestGetOnlineFeaturesSqlite(t *testing.T) { func TestGetOnlineFeaturesSqliteWithLogging(t *testing.T) { ctx := context.Background() // Pregenerated using `feast init`. - dir := "./logging/" + dir := "logging/" err := test.SetupFeatureRepoFromInitializedRepo(dir) assert.Nil(t, err) - defer test.CleanUpRepo(dir) client, closer := getClient(ctx, "file", dir, true) defer closer() entities := make(map[string]*types.RepeatedValue) @@ -204,7 +203,7 @@ func TestGetOnlineFeaturesSqliteWithLogging(t *testing.T) { // TODO(kevjumba): implement for timestamp and status expectedLogValues, _, _ := GetExpectedLogRows(featureNames, response.Results[len(request.Entities):]) expectedLogValues["driver_id"] = entities["driver_id"] - logPath, err := filepath.Abs(filepath.Join(dir, "log.parquet")) + logPath, err := filepath.Abs(filepath.Join(dir, "feature_repo", "log.parquet")) // Wait for logger to flush. assert.Eventually(t, func() bool { var _, err = os.Stat(logPath) diff --git a/go/internal/test/go_integration_test_utils.go b/go/internal/test/go_integration_test_utils.go index cee8d5f56b3..ef13feb12c9 100644 --- a/go/internal/test/go_integration_test_utils.go +++ b/go/internal/test/go_integration_test_utils.go @@ -126,10 +126,15 @@ func SetupFeatureCleanRepo(basePath string) error { func SetupFeatureRepoFromInitializedRepo(basePath string) error { path, err := filepath.Abs(basePath) + if err != nil { + return err + } + applyCommand := exec.Command("feast", "apply") applyCommand.Env = os.Environ() feature_repo_path, err := filepath.Abs(filepath.Join(path, "feature_repo")) if err != nil { + return err } applyCommand.Dir = feature_repo_path From 5f9a50e2532a9a16f87ea13e9d46a22738f8bf98 Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Tue, 12 Apr 2022 18:48:49 -0700 Subject: [PATCH 84/97] Remove Signed-off-by: Kevin Zhang --- go/cmd/server/server_test.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/go/cmd/server/server_test.go b/go/cmd/server/server_test.go index fd0298da8a4..4fde5303588 100644 --- a/go/cmd/server/server_test.go +++ b/go/cmd/server/server_test.go @@ -102,7 +102,6 @@ func TestGetFeastServingInfo(t *testing.T) { dir := "logging/" err := test.SetupFeatureRepoFromInitializedRepo(dir) assert.Nil(t, err) - defer test.CleanUpRepo(dir) client, closer := getClient(ctx, "", dir, false) defer closer() response, err := client.GetFeastServingInfo(ctx, &serving.GetFeastServingInfoRequest{}) @@ -116,7 +115,6 @@ func TestGetOnlineFeaturesSqlite(t *testing.T) { dir := "logging/" err := test.SetupFeatureRepoFromInitializedRepo(dir) assert.Nil(t, err) - defer test.CleanUpRepo(dir) client, closer := getClient(ctx, "", dir, false) defer closer() entities := make(map[string]*types.RepeatedValue) From 1a5d98e43dc1b8496ae7d5a63f4c2155abb7a42a Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Tue, 12 Apr 2022 19:06:06 -0700 Subject: [PATCH 85/97] Fix tests Signed-off-by: Kevin Zhang --- .../feast/onlinestore/sqliteonlinestore_test.go | 9 +++++---- go/internal/test/go_integration_test_utils.go | 12 +++--------- 2 files changed, 8 insertions(+), 13 deletions(-) diff --git a/go/internal/feast/onlinestore/sqliteonlinestore_test.go b/go/internal/feast/onlinestore/sqliteonlinestore_test.go index 9d45992d6e6..cbee9cd91c2 100644 --- a/go/internal/feast/onlinestore/sqliteonlinestore_test.go +++ b/go/internal/feast/onlinestore/sqliteonlinestore_test.go @@ -2,20 +2,21 @@ package onlinestore import ( "context" - "github.com/feast-dev/feast/go/internal/feast/registry" "path/filepath" "reflect" "testing" + "github.com/feast-dev/feast/go/internal/feast/registry" + "github.com/feast-dev/feast/go/internal/test" "github.com/feast-dev/feast/go/protos/feast/types" "github.com/stretchr/testify/assert" ) -func TestSqliteSetup(t *testing.T) { +func TestSqliteAndFeatureRepoSetup(t *testing.T) { dir := "../../test" feature_repo_path := filepath.Join(dir, "feature_repo") - err := test.SetupFeatureRepo(dir) + err := test.SetupCleanFeatureRepo(dir) assert.Nil(t, err) defer test.CleanUpRepo(dir) config, err := registry.NewRepoConfigFromFile(feature_repo_path) @@ -34,7 +35,7 @@ func TestSqliteSetup(t *testing.T) { func TestSqliteOnlineRead(t *testing.T) { dir := "../../test" feature_repo_path := filepath.Join(dir, "feature_repo") - test.SetupFeatureRepo(dir) + test.SetupCleanFeatureRepo(dir) defer test.CleanUpRepo(dir) config, err := registry.NewRepoConfigFromFile(feature_repo_path) assert.Nil(t, err) diff --git a/go/internal/test/go_integration_test_utils.go b/go/internal/test/go_integration_test_utils.go index ef13feb12c9..00f845518bd 100644 --- a/go/internal/test/go_integration_test_utils.go +++ b/go/internal/test/go_integration_test_utils.go @@ -85,7 +85,7 @@ func GetLatestFeatures(Rows []*Row, entities map[int64]bool) map[int64]*Row { return correctFeatureRows } -func SetupFeatureCleanRepo(basePath string) error { +func SetupCleanFeatureRepo(basePath string) error { cmd := exec.Command("feast", "init", "feature_repo") path, err := filepath.Abs(basePath) cmd.Env = os.Environ() @@ -105,10 +105,7 @@ func SetupFeatureCleanRepo(basePath string) error { return err } applyCommand.Dir = feature_repo_path - err = applyCommand.Run() - if err != nil { - return err - } + applyCommand.Run() t := time.Now() formattedTime := fmt.Sprintf("%d-%02d-%02dT%02d:%02d:%02d", @@ -138,10 +135,7 @@ func SetupFeatureRepoFromInitializedRepo(basePath string) error { return err } applyCommand.Dir = feature_repo_path - err = applyCommand.Run() - if err != nil { - return err - } + applyCommand.Run() t := time.Now() formattedTime := fmt.Sprintf("%d-%02d-%02dT%02d:%02d:%02d", From b4d94dc28295d3e69ddcf2b1be76231ab30be969 Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Tue, 12 Apr 2022 19:08:45 -0700 Subject: [PATCH 86/97] Fix Signed-off-by: Kevin Zhang --- go/cmd/server/logging/filelogstorage_test.go | 2 +- go/cmd/server/server_test.go | 2 +- go/internal/test/go_integration_test_utils.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go/cmd/server/logging/filelogstorage_test.go b/go/cmd/server/logging/filelogstorage_test.go index 319b560e806..7e85650ae20 100644 --- a/go/cmd/server/logging/filelogstorage_test.go +++ b/go/cmd/server/logging/filelogstorage_test.go @@ -64,6 +64,6 @@ func TestFlushToStorage(t *testing.T) { } } - err = test.CleanUpLogs(logPath) + err = test.CleanUpFiles(logPath) assert.Nil(t, err) } diff --git a/go/cmd/server/server_test.go b/go/cmd/server/server_test.go index 4fde5303588..a18cb529042 100644 --- a/go/cmd/server/server_test.go +++ b/go/cmd/server/server_test.go @@ -244,7 +244,7 @@ func TestGetOnlineFeaturesSqliteWithLogging(t *testing.T) { } } - err = test.CleanUpLogs(logPath) + err = test.CleanUpFile(logPath) assert.Nil(t, err) } diff --git a/go/internal/test/go_integration_test_utils.go b/go/internal/test/go_integration_test_utils.go index 00f845518bd..22a2471a77e 100644 --- a/go/internal/test/go_integration_test_utils.go +++ b/go/internal/test/go_integration_test_utils.go @@ -177,7 +177,7 @@ func GetProtoFromRecord(rec array.Record) (map[string]*types.RepeatedValue, erro return r, nil } -func CleanUpLogs(absPath string) error { +func CleanUpFile(absPath string) error { return os.Remove(absPath) } From 8d9d0f91a48042fc207ab5d57aa9e7d1242d1256 Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Tue, 12 Apr 2022 19:27:01 -0700 Subject: [PATCH 87/97] Fix tests Signed-off-by: Kevin Zhang --- .gitignore | 2 -- .../logging/feature_repo/data/online_store.db | Bin 0 -> 16384 bytes go/cmd/server/logging/filelogstorage_test.go | 3 ++- go/cmd/server/server_test.go | 6 +++--- go/internal/test/go_integration_test_utils.go | 16 ++++------------ 5 files changed, 9 insertions(+), 18 deletions(-) create mode 100644 go/cmd/server/logging/feature_repo/data/online_store.db diff --git a/.gitignore b/.gitignore index 477d2411fc2..8cf5d718f88 100644 --- a/.gitignore +++ b/.gitignore @@ -105,8 +105,6 @@ coverage.xml .hypothesis/ .pytest_cache/ infra/scripts/*.conf -go/cmd/server/logging/feature_repo/data/online_store.db -go/cmd/server/logging/feature_repo/data/registry.db # Translations *.mo diff --git a/go/cmd/server/logging/feature_repo/data/online_store.db b/go/cmd/server/logging/feature_repo/data/online_store.db new file mode 100644 index 0000000000000000000000000000000000000000..208f83ebe92acda2b4f57c55c2578b615cdae68e GIT binary patch literal 16384 zcmeI%O=uHA6ae7e>^A>9qaN%*OXnck(pHnyqOH2M)qusO)wTyywCmce8k;n6vkevz ztawon1P?{QlNAr%yjT&$gMxSzL_CRzhx&66FCLwZiAiK<;zB(LZ^@FMH+gUJWfC$k z6T009sH0T2KI5C8!X009sHfqx^g5S4t*;|XzLG(DR#&Yd)LYtA$@)5zzvl$kCX zrgl0vXJ+QLf~8vp&6sI4MBX(CDrWPYWZIMxUT}Sd_IAzVZ7w84q4bUw&SvNy zUDM~RTTfzjBYnLhf=;g*hxM4lK8QuI8Q$aacz4s{_k7udBHy{T!B3{tC;=wp%V)d>j@#b(N+2%073wr@S1NFAgks zU4Dp~I4q_76_!VLmfcN#Qs%JOsjDpcGrcZXi^O61?7;HW<%h__VJYRWu*mP<>~^&f zCq)j6ow~}h=jBCr)mHYf-@8YJUEHC|6lNE% z?HPsG#q0V%L3VM6UQU2rytXsrXBV&QviR7=9XcjncJbQ&M-#hvT`xms7kB7ONbKUZ zT?P*w)*5hJNOc{9(*7Uy3+Od^h;E|u=osq9AMkU04`0DEIDxm}wdnh*E^$Z(0w4ea zAOHd&00JNY0w4eaAn^YQv`8wuv4FWz#EmMgPs%%&5KJ Y_j036+^Ecq+M9Wa8})3IRCXiaCvQdq!~g&Q literal 0 HcmV?d00001 diff --git a/go/cmd/server/logging/filelogstorage_test.go b/go/cmd/server/logging/filelogstorage_test.go index 7e85650ae20..1da7dd38ad2 100644 --- a/go/cmd/server/logging/filelogstorage_test.go +++ b/go/cmd/server/logging/filelogstorage_test.go @@ -64,6 +64,7 @@ func TestFlushToStorage(t *testing.T) { } } - err = test.CleanUpFiles(logPath) + err = test.CleanUpFile(logPath) assert.Nil(t, err) + } diff --git a/go/cmd/server/server_test.go b/go/cmd/server/server_test.go index a18cb529042..c8863967243 100644 --- a/go/cmd/server/server_test.go +++ b/go/cmd/server/server_test.go @@ -100,7 +100,7 @@ func TestGetFeastServingInfo(t *testing.T) { ctx := context.Background() // Pregenerated using `feast init`. dir := "logging/" - err := test.SetupFeatureRepoFromInitializedRepo(dir) + err := test.MaterializeInInitializedRepo(dir) assert.Nil(t, err) client, closer := getClient(ctx, "", dir, false) defer closer() @@ -113,7 +113,7 @@ func TestGetOnlineFeaturesSqlite(t *testing.T) { ctx := context.Background() // Pregenerated using `feast init`. dir := "logging/" - err := test.SetupFeatureRepoFromInitializedRepo(dir) + err := test.MaterializeInInitializedRepo(dir) assert.Nil(t, err) client, closer := getClient(ctx, "", dir, false) defer closer() @@ -170,7 +170,7 @@ func TestGetOnlineFeaturesSqliteWithLogging(t *testing.T) { ctx := context.Background() // Pregenerated using `feast init`. dir := "logging/" - err := test.SetupFeatureRepoFromInitializedRepo(dir) + err := test.MaterializeInInitializedRepo(dir) assert.Nil(t, err) client, closer := getClient(ctx, "file", dir, true) defer closer() diff --git a/go/internal/test/go_integration_test_utils.go b/go/internal/test/go_integration_test_utils.go index 22a2471a77e..b23d4c5c79d 100644 --- a/go/internal/test/go_integration_test_utils.go +++ b/go/internal/test/go_integration_test_utils.go @@ -121,21 +121,15 @@ func SetupCleanFeatureRepo(basePath string) error { return nil } -func SetupFeatureRepoFromInitializedRepo(basePath string) error { +func MaterializeInInitializedRepo(basePath string) error { path, err := filepath.Abs(basePath) if err != nil { return err } - - applyCommand := exec.Command("feast", "apply") - applyCommand.Env = os.Environ() - feature_repo_path, err := filepath.Abs(filepath.Join(path, "feature_repo")) + feature_repo_path, err := filepath.Abs(filepath.Join(path, "feature_repo/")) if err != nil { - return err } - applyCommand.Dir = feature_repo_path - applyCommand.Run() t := time.Now() formattedTime := fmt.Sprintf("%d-%02d-%02dT%02d:%02d:%02d", @@ -144,10 +138,8 @@ func SetupFeatureRepoFromInitializedRepo(basePath string) error { materializeCommand := exec.Command("feast", "materialize-incremental", formattedTime) materializeCommand.Env = os.Environ() materializeCommand.Dir = feature_repo_path - err = materializeCommand.Run() - if err != nil { - return err - } + materializeCommand.Start() + materializeCommand.Wait() return nil } From 4e65987508973eda08623c88de9bcbe816b80940 Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Tue, 12 Apr 2022 19:54:39 -0700 Subject: [PATCH 88/97] Fix Signed-off-by: Kevin Zhang --- .../logging/feature_repo/data/online_store.db | Bin 16384 -> 0 bytes go/cmd/server/server_test.go | 6 +++--- go/internal/test/go_integration_test_utils.go | 16 ++++++++++++++-- 3 files changed, 17 insertions(+), 5 deletions(-) delete mode 100644 go/cmd/server/logging/feature_repo/data/online_store.db diff --git a/go/cmd/server/logging/feature_repo/data/online_store.db b/go/cmd/server/logging/feature_repo/data/online_store.db deleted file mode 100644 index 208f83ebe92acda2b4f57c55c2578b615cdae68e..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 16384 zcmeI%O=uHA6ae7e>^A>9qaN%*OXnck(pHnyqOH2M)qusO)wTyywCmce8k;n6vkevz ztawon1P?{QlNAr%yjT&$gMxSzL_CRzhx&66FCLwZiAiK<;zB(LZ^@FMH+gUJWfC$k z6T009sH0T2KI5C8!X009sHfqx^g5S4t*;|XzLG(DR#&Yd)LYtA$@)5zzvl$kCX zrgl0vXJ+QLf~8vp&6sI4MBX(CDrWPYWZIMxUT}Sd_IAzVZ7w84q4bUw&SvNy zUDM~RTTfzjBYnLhf=;g*hxM4lK8QuI8Q$aacz4s{_k7udBHy{T!B3{tC;=wp%V)d>j@#b(N+2%073wr@S1NFAgks zU4Dp~I4q_76_!VLmfcN#Qs%JOsjDpcGrcZXi^O61?7;HW<%h__VJYRWu*mP<>~^&f zCq)j6ow~}h=jBCr)mHYf-@8YJUEHC|6lNE% z?HPsG#q0V%L3VM6UQU2rytXsrXBV&QviR7=9XcjncJbQ&M-#hvT`xms7kB7ONbKUZ zT?P*w)*5hJNOc{9(*7Uy3+Od^h;E|u=osq9AMkU04`0DEIDxm}wdnh*E^$Z(0w4ea zAOHd&00JNY0w4eaAn^YQv`8wuv4FWz#EmMgPs%%&5KJ Y_j036+^Ecq+M9Wa8})3IRCXiaCvQdq!~g&Q diff --git a/go/cmd/server/server_test.go b/go/cmd/server/server_test.go index c8863967243..56144a84316 100644 --- a/go/cmd/server/server_test.go +++ b/go/cmd/server/server_test.go @@ -100,7 +100,7 @@ func TestGetFeastServingInfo(t *testing.T) { ctx := context.Background() // Pregenerated using `feast init`. dir := "logging/" - err := test.MaterializeInInitializedRepo(dir) + err := test.SetupInitializedRepo(dir) assert.Nil(t, err) client, closer := getClient(ctx, "", dir, false) defer closer() @@ -113,7 +113,7 @@ func TestGetOnlineFeaturesSqlite(t *testing.T) { ctx := context.Background() // Pregenerated using `feast init`. dir := "logging/" - err := test.MaterializeInInitializedRepo(dir) + err := test.SetupInitializedRepo(dir) assert.Nil(t, err) client, closer := getClient(ctx, "", dir, false) defer closer() @@ -170,7 +170,7 @@ func TestGetOnlineFeaturesSqliteWithLogging(t *testing.T) { ctx := context.Background() // Pregenerated using `feast init`. dir := "logging/" - err := test.MaterializeInInitializedRepo(dir) + err := test.SetupInitializedRepo(dir) assert.Nil(t, err) client, closer := getClient(ctx, "file", dir, true) defer closer() diff --git a/go/internal/test/go_integration_test_utils.go b/go/internal/test/go_integration_test_utils.go index b23d4c5c79d..4437b63e75a 100644 --- a/go/internal/test/go_integration_test_utils.go +++ b/go/internal/test/go_integration_test_utils.go @@ -1,6 +1,7 @@ package test import ( + "bytes" "context" "fmt" @@ -121,12 +122,22 @@ func SetupCleanFeatureRepo(basePath string) error { return nil } -func MaterializeInInitializedRepo(basePath string) error { +func SetupInitializedRepo(basePath string) error { path, err := filepath.Abs(basePath) if err != nil { return err } - feature_repo_path, err := filepath.Abs(filepath.Join(path, "feature_repo/")) + applyCommand := exec.Command("feast", "apply") + applyCommand.Env = os.Environ() + feature_repo_path, err := filepath.Abs(filepath.Join(path, "feature_repo")) + if err != nil { + return err + } + var stderr bytes.Buffer + + applyCommand.Dir = feature_repo_path + applyCommand.Stderr = &stderr + err = applyCommand.Run() if err != nil { return err } @@ -135,6 +146,7 @@ func MaterializeInInitializedRepo(basePath string) error { formattedTime := fmt.Sprintf("%d-%02d-%02dT%02d:%02d:%02d", t.Year(), t.Month(), t.Day(), t.Hour(), t.Minute(), t.Second()) + materializeCommand := exec.Command("feast", "materialize-incremental", formattedTime) materializeCommand.Env = os.Environ() materializeCommand.Dir = feature_repo_path From 9d4effa16f81e9948cf54232555a522db625f0a5 Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Tue, 12 Apr 2022 20:54:51 -0700 Subject: [PATCH 89/97] Text Signed-off-by: Kevin Zhang --- go/internal/test/go_integration_test_utils.go | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/go/internal/test/go_integration_test_utils.go b/go/internal/test/go_integration_test_utils.go index 4437b63e75a..220051dc7dd 100644 --- a/go/internal/test/go_integration_test_utils.go +++ b/go/internal/test/go_integration_test_utils.go @@ -1,9 +1,9 @@ package test import ( - "bytes" "context" "fmt" + "log" "github.com/apache/arrow/go/v8/arrow/memory" "google.golang.org/protobuf/types/known/durationpb" @@ -127,19 +127,20 @@ func SetupInitializedRepo(basePath string) error { if err != nil { return err } - applyCommand := exec.Command("feast", "apply") + applyCommand := exec.Command("/bin/sh", "-c", "feast apply") applyCommand.Env = os.Environ() feature_repo_path, err := filepath.Abs(filepath.Join(path, "feature_repo")) if err != nil { return err } - var stderr bytes.Buffer - + // var stderr bytes.Buffer + // var stdout bytes.Buffer applyCommand.Dir = feature_repo_path - applyCommand.Stderr = &stderr - err = applyCommand.Run() + out, err := applyCommand.Output() if err != nil { + log.Println(out) return err + } t := time.Now() From e604750944b29fd8dc9d16fd0957b08ae8256291 Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Tue, 12 Apr 2022 21:03:37 -0700 Subject: [PATCH 90/97] Fix Signed-off-by: Kevin Zhang --- go/cmd/server/server_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/go/cmd/server/server_test.go b/go/cmd/server/server_test.go index 56144a84316..93f471396a4 100644 --- a/go/cmd/server/server_test.go +++ b/go/cmd/server/server_test.go @@ -2,6 +2,7 @@ package main import ( "context" + "log" "net" "os" "path/filepath" @@ -172,6 +173,7 @@ func TestGetOnlineFeaturesSqliteWithLogging(t *testing.T) { dir := "logging/" err := test.SetupInitializedRepo(dir) assert.Nil(t, err) + log.Println("setup correctly") client, closer := getClient(ctx, "file", dir, true) defer closer() entities := make(map[string]*types.RepeatedValue) From 49fb8bc2b536afdac8a1bcc141d57b35dc38e61b Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Tue, 12 Apr 2022 21:19:32 -0700 Subject: [PATCH 91/97] Fix? Signed-off-by: Kevin Zhang --- .gitignore | 1 + go/cmd/server/logging/feature_repo/example.py | 20 +++++++++---------- go/internal/test/go_integration_test_utils.go | 11 ++++++---- 3 files changed, 18 insertions(+), 14 deletions(-) diff --git a/.gitignore b/.gitignore index 8cf5d718f88..822efd4fbdb 100644 --- a/.gitignore +++ b/.gitignore @@ -105,6 +105,7 @@ coverage.xml .hypothesis/ .pytest_cache/ infra/scripts/*.conf +go/cmd/server/logging/feature_repo/data/online_store.db # Translations *.mo diff --git a/go/cmd/server/logging/feature_repo/example.py b/go/cmd/server/logging/feature_repo/example.py index aa933a11dfd..ef306501fdf 100644 --- a/go/cmd/server/logging/feature_repo/example.py +++ b/go/cmd/server/logging/feature_repo/example.py @@ -1,15 +1,15 @@ # This is an example feature definition file -from datetime import timedelta +from google.protobuf.duration_pb2 import Duration -from feast import Entity, FeatureView, Field, FileSource, Float32, Int64, ValueType, FeatureService +from feast import Entity, Feature, FeatureView, FileSource, ValueType, FeatureService # Read data from parquet files. Parquet is convenient for local development mode. For # production, you can use your favorite DWH, such as BigQuery. See Feast documentation # for more info. driver_hourly_stats = FileSource( - path="/Users/kevinzhang/tecton-ai/offline_store/feast/go/cmd/server/logging/feature_repo/data/driver_stats.parquet", - timestamp_field="event_timestamp", + path="./data/driver_stats.parquet", + event_timestamp_column="event_timestamp", created_timestamp_column="created", ) @@ -23,11 +23,11 @@ driver_hourly_stats_view = FeatureView( name="driver_hourly_stats", entities=["driver_id"], - ttl=timedelta(days=1), - schema=[ - Field(name="conv_rate", dtype=Float32), - Field(name="acc_rate", dtype=Float32), - Field(name="avg_daily_trips", dtype=Int64), + ttl=Duration(seconds=86400 * 1), + features=[ + Feature(name="conv_rate", dtype=ValueType.FLOAT), + Feature(name="acc_rate", dtype=ValueType.FLOAT), + Feature(name="avg_daily_trips", dtype=ValueType.INT64), ], online=True, batch_source=driver_hourly_stats, @@ -37,4 +37,4 @@ driver_stats_fs = FeatureService( name="test_service", features=[driver_hourly_stats_view] -) +) \ No newline at end of file diff --git a/go/internal/test/go_integration_test_utils.go b/go/internal/test/go_integration_test_utils.go index 220051dc7dd..6891c81fd28 100644 --- a/go/internal/test/go_integration_test_utils.go +++ b/go/internal/test/go_integration_test_utils.go @@ -136,9 +136,8 @@ func SetupInitializedRepo(basePath string) error { // var stderr bytes.Buffer // var stdout bytes.Buffer applyCommand.Dir = feature_repo_path - out, err := applyCommand.Output() + err = applyCommand.Run() if err != nil { - log.Println(out) return err } @@ -151,8 +150,12 @@ func SetupInitializedRepo(basePath string) error { materializeCommand := exec.Command("feast", "materialize-incremental", formattedTime) materializeCommand.Env = os.Environ() materializeCommand.Dir = feature_repo_path - materializeCommand.Start() - materializeCommand.Wait() + out, err := materializeCommand.Output() + if err != nil { + log.Println(feature_repo_path) + log.Println(out) + return err + } return nil } From 72b1700d80039f153348c29ea05a284acb1912aa Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Tue, 12 Apr 2022 21:52:43 -0700 Subject: [PATCH 92/97] Fix Signed-off-by: Kevin Zhang --- go/cmd/server/logging/logging.go | 31 ++++++++++--------- go/cmd/server/logging/logging_test.go | 14 ++++----- go/cmd/server/server_test.go | 2 -- go/embedded/online_features.go | 4 +-- go/internal/feast/featurestore.go | 6 ++-- go/internal/feast/model/featureview.go | 16 ++++++---- go/internal/feast/onlineserving/serving.go | 9 +++--- go/internal/test/go_integration_test_utils.go | 9 +++--- 8 files changed, 48 insertions(+), 43 deletions(-) diff --git a/go/cmd/server/logging/logging.go b/go/cmd/server/logging/logging.go index c8dd0943f8f..b229b39f15a 100644 --- a/go/cmd/server/logging/logging.go +++ b/go/cmd/server/logging/logging.go @@ -123,11 +123,11 @@ func (s *LoggingService) flushLogsToOfflineStorage(t time.Time) error { return fmt.Errorf("could not get offline storage type for config: %s", s.fs.GetRepoConfig().OfflineStore) } if offlineStoreType == "file" { - entities, entityMap, featureViews, odfvs, err := s.GetFcos() + entityMap, featureViews, odfvs, err := s.GetFcos() if err != nil { return err } - schema, err := GetSchemaFromFeatureService(s.memoryBuffer.featureService, entities, entityMap, featureViews, odfvs) + schema, err := GetSchemaFromFeatureService(s.memoryBuffer.featureService, entityMap, featureViews, odfvs) if err != nil { return err } @@ -254,7 +254,7 @@ type Schema struct { FeaturesTypes map[string]types.ValueType_Enum } -func GetSchemaFromFeatureService(featureService *model.FeatureService, entities []*model.Entity, entityMap map[string]*model.Entity, featureViews []*model.FeatureView, onDemandFeatureViews []*model.OnDemandFeatureView) (*Schema, error) { +func GetSchemaFromFeatureService(featureService *model.FeatureService, entityMap map[string]*model.Entity, featureViews []*model.FeatureView, onDemandFeatureViews []*model.OnDemandFeatureView) (*Schema, error) { fvs := make(map[string]*model.FeatureView) odFvs := make(map[string]*model.OnDemandFeatureView) @@ -262,9 +262,10 @@ func GetSchemaFromFeatureService(featureService *model.FeatureService, entities // All joinkeys in the featureService are put in here joinKeysSet := make(map[string]interface{}) entityJoinKeyToType := make(map[string]types.ValueType_Enum) - + var entities []string for _, featureView := range featureViews { fvs[featureView.Base.Name] = featureView + entities = featureView.Entities } for _, onDemandFeatureView := range onDemandFeatureViews { @@ -282,7 +283,7 @@ func GetSchemaFromFeatureService(featureService *model.FeatureService, entities features = append(features, GetFullFeatureName(featureViewName, f.Name)) allFeatureTypes[GetFullFeatureName(featureViewName, f.Name)] = f.Dtype } - for entityName := range fv.Entities { + for entityName := range fv.EntitiesMap { entity := entityMap[entityName] if joinKeyAlias, ok := featureProjection.JoinKeyMap[entity.JoinKey]; ok { joinKeysSet[joinKeyAlias] = nil @@ -303,9 +304,9 @@ func GetSchemaFromFeatureService(featureService *model.FeatureService, entities // Only get entities in the current feature service. for _, entity := range entities { - if _, ok := joinKeysSet[entity.Name]; ok { - joinKeys = append(joinKeys, entity.JoinKey) - entityJoinKeyToType[entity.JoinKey] = entity.ValueType + if _, ok := joinKeysSet[entity]; ok { + joinKeys = append(joinKeys, entityMap[entity].JoinKey) + entityJoinKeyToType[entityMap[entity].JoinKey] = entityMap[entity].ValueType } } @@ -322,24 +323,24 @@ func GetFullFeatureName(featureViewName string, featureName string) string { return fmt.Sprintf("%s__%s", featureViewName, featureName) } -func (s *LoggingService) GetFcos() ([]*model.Entity, map[string]*model.Entity, []*model.FeatureView, []*model.OnDemandFeatureView, error) { +func (s *LoggingService) GetFcos() (map[string]*model.Entity, []*model.FeatureView, []*model.OnDemandFeatureView, error) { odfvs, err := s.fs.ListOnDemandFeatureViews() if err != nil { - return nil, nil, nil, nil, err + return nil, nil, nil, err } fvs, err := s.fs.ListFeatureViews() if err != nil { - return nil, nil, nil, nil, err + return nil, nil, nil, err } entities, err := s.fs.ListEntities(true) if err != nil { - return nil, nil, nil, nil, err + return nil, nil, nil, err } entityMap := make(map[string]*model.Entity) for _, entity := range entities { entityMap[entity.Name] = entity } - return entities, entityMap, fvs, odfvs, nil + return entityMap, fvs, odfvs, nil } func (l *LoggingService) GenerateLogs(featureService *model.FeatureService, joinKeyToEntityValues map[string][]*types.Value, features []*serving.GetOnlineFeaturesResponse_FeatureVector, requestData map[string]*types.RepeatedValue, requestId string) error { @@ -347,11 +348,11 @@ func (l *LoggingService) GenerateLogs(featureService *model.FeatureService, join return nil } - entities, entitySet, featureViews, odfvs, err := l.GetFcos() + entitySet, featureViews, odfvs, err := l.GetFcos() if err != nil { return err } - schema, err := GetSchemaFromFeatureService(featureService, entities, entitySet, featureViews, odfvs) + schema, err := GetSchemaFromFeatureService(featureService, entitySet, featureViews, odfvs) if err != nil { return err diff --git a/go/cmd/server/logging/logging_test.go b/go/cmd/server/logging/logging_test.go index 9181922a47f..77dc3501dc5 100644 --- a/go/cmd/server/logging/logging_test.go +++ b/go/cmd/server/logging/logging_test.go @@ -59,7 +59,7 @@ func TestSchemaTypeRetrieval(t *testing.T) { } } - schema, err := GetSchemaFromFeatureService(featureService, entities, entityMap, featureViews, odfvs) + schema, err := GetSchemaFromFeatureService(featureService, entityMap, featureViews, odfvs) assert.Nil(t, err) assert.Equal(t, expectedFeatureNames, schema.Features) @@ -80,13 +80,13 @@ func TestSchemaRetrievalIgnoresEntitiesNotInFeatureService(t *testing.T) { featureService, entities, featureViews, odfvs := InitializeFeatureRepoVariablesForTest() //Remove entities in featureservice for _, featureView := range featureViews { - featureView.Entities = make(map[string]struct{}) + featureView.EntitiesMap = make(map[string]struct{}) } entityMap := make(map[string]*model.Entity) for _, entity := range entities { entityMap[entity.Name] = entity } - schema, err := GetSchemaFromFeatureService(featureService, entities, entityMap, featureViews, odfvs) + schema, err := GetSchemaFromFeatureService(featureService, entityMap, featureViews, odfvs) assert.Nil(t, err) assert.Empty(t, schema.EntityTypes) } @@ -128,7 +128,7 @@ func TestSchemaUsesOrderInFeatureService(t *testing.T) { }) } - schema, err := GetSchemaFromFeatureService(featureService, entities, entityMap, featureViews, odfvs) + schema, err := GetSchemaFromFeatureService(featureService, entityMap, featureViews, odfvs) assert.Nil(t, err) // Ensure the same results @@ -196,7 +196,7 @@ func InitializeFeatureRepoVariablesForTest() (*model.FeatureService, []*model.En []*model.Feature{f1, f2}, projection1, ) - featureView1 := test.CreateFeatureView(baseFeatureView1, nil, map[string]struct{}{"driver_id": {}}) + featureView1 := test.CreateFeatureView(baseFeatureView1, nil, []string{"driver_id"}, map[string]struct{}{"driver_id": {}}) entity1 := test.CreateNewEntity("driver_id", types.ValueType_INT64, "driver_id") f3 := test.CreateNewFeature( "int32", @@ -217,7 +217,7 @@ func InitializeFeatureRepoVariablesForTest() (*model.FeatureService, []*model.En []*model.Feature{f3, f4}, projection2, ) - featureView2 := test.CreateFeatureView(baseFeatureView2, nil, map[string]struct{}{"driver_id": {}}) + featureView2 := test.CreateFeatureView(baseFeatureView2, nil, []string{"driver_id"}, map[string]struct{}{"driver_id": {}}) f5 := test.CreateNewFeature( "odfv_f1", @@ -257,7 +257,7 @@ func GetTestArrowTableAndExpectedResults() (array.Table, map[string]arrow.DataTy for _, entity := range entities { entityMap[entity.Name] = entity } - schema, err := GetSchemaFromFeatureService(featureService, entities, entityMap, featureViews, odfvs) + schema, err := GetSchemaFromFeatureService(featureService, entityMap, featureViews, odfvs) if err != nil { return nil, nil, nil, err } diff --git a/go/cmd/server/server_test.go b/go/cmd/server/server_test.go index 93f471396a4..56144a84316 100644 --- a/go/cmd/server/server_test.go +++ b/go/cmd/server/server_test.go @@ -2,7 +2,6 @@ package main import ( "context" - "log" "net" "os" "path/filepath" @@ -173,7 +172,6 @@ func TestGetOnlineFeaturesSqliteWithLogging(t *testing.T) { dir := "logging/" err := test.SetupInitializedRepo(dir) assert.Nil(t, err) - log.Println("setup correctly") client, closer := getClient(ctx, "file", dir, true) defer closer() entities := make(map[string]*types.RepeatedValue) diff --git a/go/embedded/online_features.go b/go/embedded/online_features.go index 104d171cac6..0782f25d7e2 100644 --- a/go/embedded/online_features.go +++ b/go/embedded/online_features.go @@ -70,7 +70,7 @@ func (s *OnlineFeatureService) GetEntityTypesMap(featureRefs []string) (map[stri // skip on demand feature views continue } - for entityName := range view.Entities { + for entityName := range view.EntitiesMap { entity := entitiesByName[entityName] joinKeyTypes[entity.JoinKey] = int32(entity.ValueType.Number()) } @@ -99,7 +99,7 @@ func (s *OnlineFeatureService) GetEntityTypesMapByFeatureService(featureServiceN // skip on demand feature views continue } - for entityName := range view.Entities { + for entityName := range view.EntitiesMap { entity := entitiesByName[entityName] joinKeyTypes[entity.JoinKey] = int32(entity.ValueType.Number()) } diff --git a/go/internal/feast/featurestore.go b/go/internal/feast/featurestore.go index b165ca38c5b..745f4ca3408 100644 --- a/go/internal/feast/featurestore.go +++ b/go/internal/feast/featurestore.go @@ -122,7 +122,7 @@ func (fs *FeatureStore) GetOnlineFeatures( entitylessCase := false for _, featureView := range featureViews { - if _, ok := featureView.Entities[model.DUMMY_ENTITY_NAME]; ok { + if _, ok := featureView.EntitiesMap[model.DUMMY_ENTITY_NAME]; ok { entitylessCase = true break } @@ -270,8 +270,8 @@ func (fs *FeatureStore) GetFeatureView(featureViewName string, hideDummyEntity b if err != nil { return nil, err } - if _, ok := fv.Entities[model.DUMMY_ENTITY_NAME]; ok && hideDummyEntity { - fv.Entities = make(map[string]struct{}) + if _, ok := fv.EntitiesMap[model.DUMMY_ENTITY_NAME]; ok && hideDummyEntity { + fv.EntitiesMap = make(map[string]struct{}) } return fv, nil } diff --git a/go/internal/feast/model/featureview.go b/go/internal/feast/model/featureview.go index bdb86f0ace5..3d2ef117e27 100644 --- a/go/internal/feast/model/featureview.go +++ b/go/internal/feast/model/featureview.go @@ -18,7 +18,8 @@ type FeatureView struct { Base *BaseFeatureView Ttl *durationpb.Duration // Make entities set so that search for dummy entity is faster - Entities map[string]struct{} + EntitiesMap map[string]struct{} + Entities []string } func NewFeatureViewFromProto(proto *core.FeatureView) *FeatureView { @@ -26,12 +27,14 @@ func NewFeatureViewFromProto(proto *core.FeatureView) *FeatureView { Ttl: &(*proto.Spec.Ttl), } if len(proto.Spec.Entities) == 0 { - featureView.Entities = map[string]struct{}{DUMMY_ENTITY_NAME: {}} + featureView.EntitiesMap = map[string]struct{}{DUMMY_ENTITY_NAME: {}} + featureView.Entities = []string{} } else { - featureView.Entities = make(map[string]struct{}) + featureView.EntitiesMap = make(map[string]struct{}) for _, entityName := range proto.Spec.Entities { - featureView.Entities[entityName] = struct{}{} + featureView.EntitiesMap[entityName] = struct{}{} } + featureView.Entities = proto.Spec.Entities } return featureView } @@ -39,8 +42,9 @@ func NewFeatureViewFromProto(proto *core.FeatureView) *FeatureView { func (fs *FeatureView) NewFeatureViewFromBase(base *BaseFeatureView) *FeatureView { ttl := durationpb.Duration{Seconds: fs.Ttl.Seconds, Nanos: fs.Ttl.Nanos} featureView := &FeatureView{Base: base, - Ttl: &ttl, - Entities: fs.Entities, + Ttl: &ttl, + EntitiesMap: fs.EntitiesMap, + Entities: fs.Entities, } return featureView } diff --git a/go/internal/feast/onlineserving/serving.go b/go/internal/feast/onlineserving/serving.go index ebf0f3f324f..17392997f08 100644 --- a/go/internal/feast/onlineserving/serving.go +++ b/go/internal/feast/onlineserving/serving.go @@ -4,6 +4,9 @@ import ( "crypto/sha256" "errors" "fmt" + "sort" + "strings" + "github.com/apache/arrow/go/v8/arrow" "github.com/apache/arrow/go/v8/arrow/memory" "github.com/feast-dev/feast/go/internal/feast/model" @@ -14,8 +17,6 @@ import ( "github.com/golang/protobuf/proto" "google.golang.org/protobuf/types/known/durationpb" "google.golang.org/protobuf/types/known/timestamppb" - "sort" - "strings" ) /* @@ -249,7 +250,7 @@ func GetEntityMaps(requestedFeatureViews []*FeatureViewAndRefs, entities []*mode joinKeyToAliasMap = map[string]string{} } - for entityName := range featureView.Entities { + for entityName := range featureView.EntitiesMap { joinKey := entitiesByName[entityName].JoinKey entityNameToJoinKeyMap[entityName] = joinKey @@ -516,7 +517,7 @@ func GroupFeatureRefs(requestedFeatureViews []*FeatureViewAndRefs, joinKeys := make([]string, 0) fv := featuresAndView.View featureNames := featuresAndView.FeatureRefs - for entity := range fv.Entities { + for entity := range fv.EntitiesMap { joinKeys = append(joinKeys, entityNameToJoinKeyMap[entity]) } diff --git a/go/internal/test/go_integration_test_utils.go b/go/internal/test/go_integration_test_utils.go index 6891c81fd28..6a7b4518e30 100644 --- a/go/internal/test/go_integration_test_utils.go +++ b/go/internal/test/go_integration_test_utils.go @@ -229,10 +229,11 @@ func CreateNewFeatureViewProjection(name string, nameAlias string, features []*m } } -func CreateFeatureView(base *model.BaseFeatureView, ttl *durationpb.Duration, entities map[string]struct{}) *model.FeatureView { +func CreateFeatureView(base *model.BaseFeatureView, ttl *durationpb.Duration, entities []string, entitiesMap map[string]struct{}) *model.FeatureView { return &model.FeatureView{ - Base: base, - Ttl: ttl, - Entities: entities, + Base: base, + Ttl: ttl, + Entities: entities, + EntitiesMap: entitiesMap, } } From b4f7f416951d51a49bba3e82327e54382e88680f Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Tue, 12 Apr 2022 22:00:40 -0700 Subject: [PATCH 93/97] Fix Signed-off-by: Kevin Zhang --- .../feast/onlineserving/serving_test.go | 25 ++++++++++--------- 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/go/internal/feast/onlineserving/serving_test.go b/go/internal/feast/onlineserving/serving_test.go index d33ec87b820..597b8f0d30f 100644 --- a/go/internal/feast/onlineserving/serving_test.go +++ b/go/internal/feast/onlineserving/serving_test.go @@ -1,13 +1,14 @@ package onlineserving import ( + "testing" + "github.com/feast-dev/feast/go/internal/feast/model" "github.com/feast-dev/feast/go/protos/feast/core" "github.com/feast-dev/feast/go/protos/feast/types" "github.com/stretchr/testify/assert" "google.golang.org/protobuf/types/known/durationpb" "google.golang.org/protobuf/types/known/timestamppb" - "testing" ) func TestGroupingFeatureRefs(t *testing.T) { @@ -18,19 +19,19 @@ func TestGroupingFeatureRefs(t *testing.T) { NameAlias: "aliasViewA", }, }, - Entities: map[string]struct{}{"driver": {}, "customer": {}}, + EntitiesMap: map[string]struct{}{"driver": {}, "customer": {}}, } viewB := &model.FeatureView{ - Base: &model.BaseFeatureView{Name: "viewB"}, - Entities: map[string]struct{}{"driver": {}, "customer": {}}, + Base: &model.BaseFeatureView{Name: "viewB"}, + EntitiesMap: map[string]struct{}{"driver": {}, "customer": {}}, } viewC := &model.FeatureView{ - Base: &model.BaseFeatureView{Name: "viewC"}, - Entities: map[string]struct{}{"driver": {}}, + Base: &model.BaseFeatureView{Name: "viewC"}, + EntitiesMap: map[string]struct{}{"driver": {}}, } viewD := &model.FeatureView{ - Base: &model.BaseFeatureView{Name: "viewD"}, - Entities: map[string]struct{}{"customer": {}}, + Base: &model.BaseFeatureView{Name: "viewD"}, + EntitiesMap: map[string]struct{}{"customer": {}}, } refGroups, _ := GroupFeatureRefs( []*FeatureViewAndRefs{ @@ -103,11 +104,11 @@ func TestGroupingFeatureRefsWithJoinKeyAliases(t *testing.T) { JoinKeyMap: map[string]string{"location_id": "destination_id"}, }, }, - Entities: map[string]struct{}{"location": {}}, + EntitiesMap: map[string]struct{}{"location": {}}, } viewB := &model.FeatureView{ - Base: &model.BaseFeatureView{Name: "viewB"}, - Entities: map[string]struct{}{"location": {}}, + Base: &model.BaseFeatureView{Name: "viewB"}, + EntitiesMap: map[string]struct{}{"location": {}}, } refGroups, _ := GroupFeatureRefs( @@ -162,7 +163,7 @@ func TestGroupingFeatureRefsWithMissingKey(t *testing.T) { JoinKeyMap: map[string]string{"location_id": "destination_id"}, }, }, - Entities: map[string]struct{}{"location": {}}, + EntitiesMap: map[string]struct{}{"location": {}}, } _, err := GroupFeatureRefs( From f19d19dae6cbcf328ae4ffc50a01122fcb475d82 Mon Sep 17 00:00:00 2001 From: pyalex Date: Wed, 13 Apr 2022 10:28:13 -0700 Subject: [PATCH 94/97] remove entity map Signed-off-by: pyalex --- go/cmd/server/logging/logging.go | 2 +- go/cmd/server/logging/logging_test.go | 6 ++-- go/embedded/online_features.go | 4 +-- go/internal/feast/featurestore.go | 6 ++-- go/internal/feast/model/featureview.go | 29 ++++++++++--------- go/internal/feast/onlineserving/serving.go | 6 ++-- .../feast/onlineserving/serving_test.go | 22 +++++++------- go/internal/test/go_integration_test_utils.go | 9 +++--- 8 files changed, 42 insertions(+), 42 deletions(-) diff --git a/go/cmd/server/logging/logging.go b/go/cmd/server/logging/logging.go index b229b39f15a..010644709ae 100644 --- a/go/cmd/server/logging/logging.go +++ b/go/cmd/server/logging/logging.go @@ -283,7 +283,7 @@ func GetSchemaFromFeatureService(featureService *model.FeatureService, entityMap features = append(features, GetFullFeatureName(featureViewName, f.Name)) allFeatureTypes[GetFullFeatureName(featureViewName, f.Name)] = f.Dtype } - for entityName := range fv.EntitiesMap { + for _, entityName := range fv.Entities { entity := entityMap[entityName] if joinKeyAlias, ok := featureProjection.JoinKeyMap[entity.JoinKey]; ok { joinKeysSet[joinKeyAlias] = nil diff --git a/go/cmd/server/logging/logging_test.go b/go/cmd/server/logging/logging_test.go index 77dc3501dc5..68da0bf498f 100644 --- a/go/cmd/server/logging/logging_test.go +++ b/go/cmd/server/logging/logging_test.go @@ -80,7 +80,7 @@ func TestSchemaRetrievalIgnoresEntitiesNotInFeatureService(t *testing.T) { featureService, entities, featureViews, odfvs := InitializeFeatureRepoVariablesForTest() //Remove entities in featureservice for _, featureView := range featureViews { - featureView.EntitiesMap = make(map[string]struct{}) + featureView.Entities = []string{} } entityMap := make(map[string]*model.Entity) for _, entity := range entities { @@ -196,7 +196,7 @@ func InitializeFeatureRepoVariablesForTest() (*model.FeatureService, []*model.En []*model.Feature{f1, f2}, projection1, ) - featureView1 := test.CreateFeatureView(baseFeatureView1, nil, []string{"driver_id"}, map[string]struct{}{"driver_id": {}}) + featureView1 := test.CreateFeatureView(baseFeatureView1, nil, []string{"driver_id"}) entity1 := test.CreateNewEntity("driver_id", types.ValueType_INT64, "driver_id") f3 := test.CreateNewFeature( "int32", @@ -217,7 +217,7 @@ func InitializeFeatureRepoVariablesForTest() (*model.FeatureService, []*model.En []*model.Feature{f3, f4}, projection2, ) - featureView2 := test.CreateFeatureView(baseFeatureView2, nil, []string{"driver_id"}, map[string]struct{}{"driver_id": {}}) + featureView2 := test.CreateFeatureView(baseFeatureView2, nil, []string{"driver_id"}) f5 := test.CreateNewFeature( "odfv_f1", diff --git a/go/embedded/online_features.go b/go/embedded/online_features.go index 0782f25d7e2..24a54894306 100644 --- a/go/embedded/online_features.go +++ b/go/embedded/online_features.go @@ -70,7 +70,7 @@ func (s *OnlineFeatureService) GetEntityTypesMap(featureRefs []string) (map[stri // skip on demand feature views continue } - for entityName := range view.EntitiesMap { + for _, entityName := range view.Entities { entity := entitiesByName[entityName] joinKeyTypes[entity.JoinKey] = int32(entity.ValueType.Number()) } @@ -99,7 +99,7 @@ func (s *OnlineFeatureService) GetEntityTypesMapByFeatureService(featureServiceN // skip on demand feature views continue } - for entityName := range view.EntitiesMap { + for _, entityName := range view.Entities { entity := entitiesByName[entityName] joinKeyTypes[entity.JoinKey] = int32(entity.ValueType.Number()) } diff --git a/go/internal/feast/featurestore.go b/go/internal/feast/featurestore.go index 745f4ca3408..5e10f4978e0 100644 --- a/go/internal/feast/featurestore.go +++ b/go/internal/feast/featurestore.go @@ -122,7 +122,7 @@ func (fs *FeatureStore) GetOnlineFeatures( entitylessCase := false for _, featureView := range featureViews { - if _, ok := featureView.EntitiesMap[model.DUMMY_ENTITY_NAME]; ok { + if featureView.HasEntity(model.DUMMY_ENTITY_NAME) { entitylessCase = true break } @@ -270,8 +270,8 @@ func (fs *FeatureStore) GetFeatureView(featureViewName string, hideDummyEntity b if err != nil { return nil, err } - if _, ok := fv.EntitiesMap[model.DUMMY_ENTITY_NAME]; ok && hideDummyEntity { - fv.EntitiesMap = make(map[string]struct{}) + if fv.HasEntity(model.DUMMY_ENTITY_NAME) && hideDummyEntity { + fv.Entities = []string{} } return fv, nil } diff --git a/go/internal/feast/model/featureview.go b/go/internal/feast/model/featureview.go index 3d2ef117e27..85fc7a60eeb 100644 --- a/go/internal/feast/model/featureview.go +++ b/go/internal/feast/model/featureview.go @@ -15,11 +15,9 @@ const ( var DUMMY_ENTITY types.Value = types.Value{Val: &types.Value_StringVal{StringVal: DUMMY_ENTITY_VAL}} type FeatureView struct { - Base *BaseFeatureView - Ttl *durationpb.Duration - // Make entities set so that search for dummy entity is faster - EntitiesMap map[string]struct{} - Entities []string + Base *BaseFeatureView + Ttl *durationpb.Duration + Entities []string } func NewFeatureViewFromProto(proto *core.FeatureView) *FeatureView { @@ -27,13 +25,8 @@ func NewFeatureViewFromProto(proto *core.FeatureView) *FeatureView { Ttl: &(*proto.Spec.Ttl), } if len(proto.Spec.Entities) == 0 { - featureView.EntitiesMap = map[string]struct{}{DUMMY_ENTITY_NAME: {}} - featureView.Entities = []string{} + featureView.Entities = []string{DUMMY_ENTITY_NAME} } else { - featureView.EntitiesMap = make(map[string]struct{}) - for _, entityName := range proto.Spec.Entities { - featureView.EntitiesMap[entityName] = struct{}{} - } featureView.Entities = proto.Spec.Entities } return featureView @@ -42,9 +35,17 @@ func NewFeatureViewFromProto(proto *core.FeatureView) *FeatureView { func (fs *FeatureView) NewFeatureViewFromBase(base *BaseFeatureView) *FeatureView { ttl := durationpb.Duration{Seconds: fs.Ttl.Seconds, Nanos: fs.Ttl.Nanos} featureView := &FeatureView{Base: base, - Ttl: &ttl, - EntitiesMap: fs.EntitiesMap, - Entities: fs.Entities, + Ttl: &ttl, + Entities: fs.Entities, } return featureView } + +func (fs *FeatureView) HasEntity(lookup string) bool { + for _, entityName := range fs.Entities { + if entityName == lookup { + return true + } + } + return false +} diff --git a/go/internal/feast/onlineserving/serving.go b/go/internal/feast/onlineserving/serving.go index 17392997f08..381ba5f0f2f 100644 --- a/go/internal/feast/onlineserving/serving.go +++ b/go/internal/feast/onlineserving/serving.go @@ -250,7 +250,7 @@ func GetEntityMaps(requestedFeatureViews []*FeatureViewAndRefs, entities []*mode joinKeyToAliasMap = map[string]string{} } - for entityName := range featureView.EntitiesMap { + for _, entityName := range featureView.Entities { joinKey := entitiesByName[entityName].JoinKey entityNameToJoinKeyMap[entityName] = joinKey @@ -517,8 +517,8 @@ func GroupFeatureRefs(requestedFeatureViews []*FeatureViewAndRefs, joinKeys := make([]string, 0) fv := featuresAndView.View featureNames := featuresAndView.FeatureRefs - for entity := range fv.EntitiesMap { - joinKeys = append(joinKeys, entityNameToJoinKeyMap[entity]) + for _, entityName := range fv.Entities { + joinKeys = append(joinKeys, entityNameToJoinKeyMap[entityName]) } groupKeyBuilder := make([]string, 0) diff --git a/go/internal/feast/onlineserving/serving_test.go b/go/internal/feast/onlineserving/serving_test.go index 597b8f0d30f..2f4cf8eabaa 100644 --- a/go/internal/feast/onlineserving/serving_test.go +++ b/go/internal/feast/onlineserving/serving_test.go @@ -19,19 +19,19 @@ func TestGroupingFeatureRefs(t *testing.T) { NameAlias: "aliasViewA", }, }, - EntitiesMap: map[string]struct{}{"driver": {}, "customer": {}}, + Entities: []string{"driver", "customer"}, } viewB := &model.FeatureView{ - Base: &model.BaseFeatureView{Name: "viewB"}, - EntitiesMap: map[string]struct{}{"driver": {}, "customer": {}}, + Base: &model.BaseFeatureView{Name: "viewB"}, + Entities: []string{"driver", "customer"}, } viewC := &model.FeatureView{ - Base: &model.BaseFeatureView{Name: "viewC"}, - EntitiesMap: map[string]struct{}{"driver": {}}, + Base: &model.BaseFeatureView{Name: "viewC"}, + Entities: []string{"driver"}, } viewD := &model.FeatureView{ - Base: &model.BaseFeatureView{Name: "viewD"}, - EntitiesMap: map[string]struct{}{"customer": {}}, + Base: &model.BaseFeatureView{Name: "viewD"}, + Entities: []string{"customer"}, } refGroups, _ := GroupFeatureRefs( []*FeatureViewAndRefs{ @@ -104,11 +104,11 @@ func TestGroupingFeatureRefsWithJoinKeyAliases(t *testing.T) { JoinKeyMap: map[string]string{"location_id": "destination_id"}, }, }, - EntitiesMap: map[string]struct{}{"location": {}}, + Entities: []string{"location"}, } viewB := &model.FeatureView{ - Base: &model.BaseFeatureView{Name: "viewB"}, - EntitiesMap: map[string]struct{}{"location": {}}, + Base: &model.BaseFeatureView{Name: "viewB"}, + Entities: []string{"location"}, } refGroups, _ := GroupFeatureRefs( @@ -163,7 +163,7 @@ func TestGroupingFeatureRefsWithMissingKey(t *testing.T) { JoinKeyMap: map[string]string{"location_id": "destination_id"}, }, }, - EntitiesMap: map[string]struct{}{"location": {}}, + Entities: []string{"location"}, } _, err := GroupFeatureRefs( diff --git a/go/internal/test/go_integration_test_utils.go b/go/internal/test/go_integration_test_utils.go index 6a7b4518e30..9fcf46c3f7b 100644 --- a/go/internal/test/go_integration_test_utils.go +++ b/go/internal/test/go_integration_test_utils.go @@ -229,11 +229,10 @@ func CreateNewFeatureViewProjection(name string, nameAlias string, features []*m } } -func CreateFeatureView(base *model.BaseFeatureView, ttl *durationpb.Duration, entities []string, entitiesMap map[string]struct{}) *model.FeatureView { +func CreateFeatureView(base *model.BaseFeatureView, ttl *durationpb.Duration, entities []string) *model.FeatureView { return &model.FeatureView{ - Base: base, - Ttl: ttl, - Entities: entities, - EntitiesMap: entitiesMap, + Base: base, + Ttl: ttl, + Entities: entities, } } From 0145e3d95d8277f36ceddfa5a4aab0108f647ac9 Mon Sep 17 00:00:00 2001 From: pyalex Date: Wed, 13 Apr 2022 10:30:21 -0700 Subject: [PATCH 95/97] remove Cache method from registry Signed-off-by: pyalex --- go/internal/feast/registry/registry.go | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/go/internal/feast/registry/registry.go b/go/internal/feast/registry/registry.go index 833efee5911..38cf167a9fd 100644 --- a/go/internal/feast/registry/registry.go +++ b/go/internal/feast/registry/registry.go @@ -37,17 +37,6 @@ type Registry struct { mu sync.Mutex } -func (r *Registry) CacheFeatureService(featureService *core.FeatureService) { - r.mu.Lock() - defer r.mu.Unlock() - if r.cachedFeatureServices == nil { - r.cachedFeatureServices = make(map[string]map[string]*core.FeatureService) - } - r.cachedFeatureServices[featureService.Spec.Project] = map[string]*core.FeatureService{ - featureService.Spec.Name: featureService, - } -} - func NewRegistry(registryConfig *RegistryConfig, repoPath string) (*Registry, error) { registryStoreType := registryConfig.RegistryStoreType registryPath := registryConfig.Path From cad1156b72fe8303836180b44a6974e1c2b132c3 Mon Sep 17 00:00:00 2001 From: pyalex Date: Wed, 13 Apr 2022 11:44:11 -0700 Subject: [PATCH 96/97] clean up pre-initialized repo Signed-off-by: pyalex --- go/cmd/server/logging/feature_repo/example.py | 2 +- go/cmd/server/server_test.go | 6 +++ .../feast/onlinestore/sqliteonlinestore.go | 9 ++-- go/internal/test/go_integration_test_utils.go | 45 ++++++++++++------- 4 files changed, 42 insertions(+), 20 deletions(-) diff --git a/go/cmd/server/logging/feature_repo/example.py b/go/cmd/server/logging/feature_repo/example.py index ef306501fdf..70e5e0a11d7 100644 --- a/go/cmd/server/logging/feature_repo/example.py +++ b/go/cmd/server/logging/feature_repo/example.py @@ -23,7 +23,7 @@ driver_hourly_stats_view = FeatureView( name="driver_hourly_stats", entities=["driver_id"], - ttl=Duration(seconds=86400 * 1), + ttl=Duration(seconds=86400 * 365 * 10), features=[ Feature(name="conv_rate", dtype=ValueType.FLOAT), Feature(name="acc_rate", dtype=ValueType.FLOAT), diff --git a/go/cmd/server/server_test.go b/go/cmd/server/server_test.go index 56144a84316..9aa54777923 100644 --- a/go/cmd/server/server_test.go +++ b/go/cmd/server/server_test.go @@ -102,6 +102,8 @@ func TestGetFeastServingInfo(t *testing.T) { dir := "logging/" err := test.SetupInitializedRepo(dir) assert.Nil(t, err) + defer test.CleanUpInitializedRepo(dir) + client, closer := getClient(ctx, "", dir, false) defer closer() response, err := client.GetFeastServingInfo(ctx, &serving.GetFeastServingInfoRequest{}) @@ -115,6 +117,8 @@ func TestGetOnlineFeaturesSqlite(t *testing.T) { dir := "logging/" err := test.SetupInitializedRepo(dir) assert.Nil(t, err) + defer test.CleanUpInitializedRepo(dir) + client, closer := getClient(ctx, "", dir, false) defer closer() entities := make(map[string]*types.RepeatedValue) @@ -172,6 +176,8 @@ func TestGetOnlineFeaturesSqliteWithLogging(t *testing.T) { dir := "logging/" err := test.SetupInitializedRepo(dir) assert.Nil(t, err) + defer test.CleanUpInitializedRepo(dir) + client, closer := getClient(ctx, "file", dir, true) defer closer() entities := make(map[string]*types.RepeatedValue) diff --git a/go/internal/feast/onlinestore/sqliteonlinestore.go b/go/internal/feast/onlinestore/sqliteonlinestore.go index 09c8eb97b4c..f8c53255455 100644 --- a/go/internal/feast/onlinestore/sqliteonlinestore.go +++ b/go/internal/feast/onlinestore/sqliteonlinestore.go @@ -82,9 +82,6 @@ func (s *SqliteOnlineStore) OnlineRead(ctx context.Context, entityKeys []*types. featureNamesToIdx[name] = idx } - for idx := 0; idx < len(entityKeys); idx++ { - results[idx] = make([]FeatureData, featureCount) - } for _, featureViewName := range featureViewNames { query_string := fmt.Sprintf(`SELECT entity_key, feature_name, Value, event_ts FROM %s @@ -108,7 +105,11 @@ func (s *SqliteOnlineStore) OnlineRead(ctx context.Context, entityKeys []*types. if err := proto.Unmarshal(valueString, &value); err != nil { return nil, errors.New("error converting parsed value to types.Value") } - results[entityNameToEntityIndex[hashSerializedEntityKey(&entity_key)]][featureNamesToIdx[feature_name]] = FeatureData{Reference: serving.FeatureReferenceV2{FeatureViewName: featureViewName, FeatureName: feature_name}, + rowIdx := entityNameToEntityIndex[hashSerializedEntityKey(&entity_key)] + if results[rowIdx] == nil { + results[rowIdx] = make([]FeatureData, featureCount) + } + results[rowIdx][featureNamesToIdx[feature_name]] = FeatureData{Reference: serving.FeatureReferenceV2{FeatureViewName: featureViewName, FeatureName: feature_name}, Timestamp: *timestamppb.New(event_ts), Value: types.Value{Val: value.Val}, } diff --git a/go/internal/test/go_integration_test_utils.go b/go/internal/test/go_integration_test_utils.go index 9fcf46c3f7b..d66a5461930 100644 --- a/go/internal/test/go_integration_test_utils.go +++ b/go/internal/test/go_integration_test_utils.go @@ -101,11 +101,11 @@ func SetupCleanFeatureRepo(basePath string) error { } applyCommand := exec.Command("feast", "apply") applyCommand.Env = os.Environ() - feature_repo_path, err := filepath.Abs(filepath.Join(path, "feature_repo")) + featureRepoPath, err := filepath.Abs(filepath.Join(path, "feature_repo")) if err != nil { return err } - applyCommand.Dir = feature_repo_path + applyCommand.Dir = featureRepoPath applyCommand.Run() t := time.Now() @@ -114,11 +114,12 @@ func SetupCleanFeatureRepo(basePath string) error { t.Hour(), t.Minute(), t.Second()) materializeCommand := exec.Command("feast", "materialize-incremental", formattedTime) materializeCommand.Env = os.Environ() - materializeCommand.Dir = feature_repo_path + materializeCommand.Dir = featureRepoPath err = materializeCommand.Run() if err != nil { return err } + return nil } @@ -127,15 +128,15 @@ func SetupInitializedRepo(basePath string) error { if err != nil { return err } - applyCommand := exec.Command("/bin/sh", "-c", "feast apply") + applyCommand := exec.Command("feast", "apply") applyCommand.Env = os.Environ() - feature_repo_path, err := filepath.Abs(filepath.Join(path, "feature_repo")) + featureRepoPath, err := filepath.Abs(filepath.Join(path, "feature_repo")) if err != nil { return err } // var stderr bytes.Buffer // var stdout bytes.Buffer - applyCommand.Dir = feature_repo_path + applyCommand.Dir = featureRepoPath err = applyCommand.Run() if err != nil { return err @@ -149,26 +150,40 @@ func SetupInitializedRepo(basePath string) error { materializeCommand := exec.Command("feast", "materialize-incremental", formattedTime) materializeCommand.Env = os.Environ() - materializeCommand.Dir = feature_repo_path + materializeCommand.Dir = featureRepoPath out, err := materializeCommand.Output() if err != nil { - log.Println(feature_repo_path) - log.Println(out) + log.Println(string(out)) return err } return nil } -func CleanUpRepo(basePath string) error { - feature_repo_path, err := filepath.Abs(filepath.Join(basePath, "feature_repo")) +func CleanUpInitializedRepo(basePath string) { + featureRepoPath, err := filepath.Abs(filepath.Join(basePath, "feature_repo")) if err != nil { - return err + log.Fatal(err) } - err = os.RemoveAll(feature_repo_path) + + err = os.Remove(filepath.Join(featureRepoPath, "data", "registry.db")) if err != nil { - return fmt.Errorf("couldn't remove feature repo path %s", err) + log.Fatal(err) + } + err = os.Remove(filepath.Join(featureRepoPath, "data", "online_store.db")) + if err != nil { + log.Fatal(err) + } +} + +func CleanUpRepo(basePath string) { + featureRepoPath, err := filepath.Abs(filepath.Join(basePath, "feature_repo")) + if err != nil { + log.Fatal(err) + } + err = os.RemoveAll(featureRepoPath) + if err != nil { + log.Fatal(err) } - return nil } func GetProtoFromRecord(rec array.Record) (map[string]*types.RepeatedValue, error) { From 89960b2101de06ddd3699e07f57ed878d348b5b4 Mon Sep 17 00:00:00 2001 From: pyalex Date: Wed, 13 Apr 2022 11:57:11 -0700 Subject: [PATCH 97/97] git ignore full data directory in tests Signed-off-by: pyalex --- .gitignore | 2 +- .../feature_repo/{data => }/driver_stats.parquet | Bin go/cmd/server/logging/feature_repo/example.py | 2 +- go/cmd/server/server_test.go | 2 +- 4 files changed, 3 insertions(+), 3 deletions(-) rename go/cmd/server/logging/feature_repo/{data => }/driver_stats.parquet (100%) diff --git a/.gitignore b/.gitignore index 822efd4fbdb..0f3165e8414 100644 --- a/.gitignore +++ b/.gitignore @@ -105,7 +105,7 @@ coverage.xml .hypothesis/ .pytest_cache/ infra/scripts/*.conf -go/cmd/server/logging/feature_repo/data/online_store.db +go/cmd/server/logging/feature_repo/data/ # Translations *.mo diff --git a/go/cmd/server/logging/feature_repo/data/driver_stats.parquet b/go/cmd/server/logging/feature_repo/driver_stats.parquet similarity index 100% rename from go/cmd/server/logging/feature_repo/data/driver_stats.parquet rename to go/cmd/server/logging/feature_repo/driver_stats.parquet diff --git a/go/cmd/server/logging/feature_repo/example.py b/go/cmd/server/logging/feature_repo/example.py index 70e5e0a11d7..f3ca6123083 100644 --- a/go/cmd/server/logging/feature_repo/example.py +++ b/go/cmd/server/logging/feature_repo/example.py @@ -8,7 +8,7 @@ # production, you can use your favorite DWH, such as BigQuery. See Feast documentation # for more info. driver_hourly_stats = FileSource( - path="./data/driver_stats.parquet", + path="driver_stats.parquet", event_timestamp_column="event_timestamp", created_timestamp_column="created", ) diff --git a/go/cmd/server/server_test.go b/go/cmd/server/server_test.go index 9aa54777923..9d4ffb50bf8 100644 --- a/go/cmd/server/server_test.go +++ b/go/cmd/server/server_test.go @@ -146,7 +146,7 @@ func TestGetOnlineFeaturesSqlite(t *testing.T) { {Val: &types.Value_Int64Val{Int64Val: 1005}}, } expectedFeatureNamesResp := []string{"driver_id", "conv_rate", "acc_rate", "avg_daily_trips"} - rows, err := test.ReadParquet(filepath.Join(dir, "feature_repo", "data", "driver_stats.parquet")) + rows, err := test.ReadParquet(filepath.Join(dir, "feature_repo", "driver_stats.parquet")) assert.Nil(t, err) entityKeys := map[int64]bool{1001: true, 1003: true, 1005: true} correctFeatures := test.GetLatestFeatures(rows, entityKeys)