// 2017 Rackspace, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"): you may // not use this file except in compliance with the License. You may obtain // a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the // License for the specific language governing permissions and limitations // under the License. package mongo import ( "context" "encoding/json" "errors" "fmt" "strconv" "strings" "time" "go.mongodb.org/mongo-driver/mongo" "github.rackspace.com/DCM/tools" "github.rackspace.com/gorax/clients/dcxAutohost" "go.mongodb.org/mongo-driver/mongo/options" "go.mongodb.org/mongo-driver/bson" "github.rackspace.com/DCM/migration/pkg/tools/journal" "github.rackspace.com/gorax/clients/mongodbAPI" ) // set how long to wait for a response from mongo for large queries; sharded collections take longer const findTimeout = 90 // DeleteVariance is used to determine the variance of devices between what our database holds and what // data warehouse holds. This is a safeguard to prevent data warehouse communications or remote imports // from erroneously marking devices as deleted in our database. const DeleteVariance = 800 // CountMongo method will take one of account/device/moveGroup and an optional projection to // query Mongo and return a list of mongo.Devices. func (c *Client) CountMongo(account, device, moveGroup string) (int, error) { if c.table == "" { err := errors.New("CountMongo/no collection selected") return 0, err } query := bson.D{} if c.hideDeleted { query = append(query, DeletedNETrue) } if account != "" { x, err := strconv.Atoi(account) if err == nil { query = append(query, bson.E{"account.number", x}) } } if device != "" { x, err := strconv.Atoi(device) if err == nil { query = append(query, bson.E{"_id", x}) } } if moveGroup != "" { query = append(query, bson.E{"move_group", moveGroup}) } ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() count, err := c.db3.Database(database).Collection(c.table).CountDocuments(ctx, query) return int(count), err } // CollectDistinct will take one of account/device/moveGroup to query Mongo and return a distinct list of // values for the specified field func (c *Client) CollectDistinct(account, device, moveGroup, fieldName string) ([]string, error) { if c.table == "" { return []string{}, errors.New("CollectMongo/no collection selected") } query := bson.D{} if c.hideDeleted { query = append(query, DeletedNETrue) } if account != "" { x, err := strconv.Atoi(account) if err == nil { query = append(query, bson.E{"account.number", x}) } } if device != "" { x, err := strconv.Atoi(device) if err == nil { query = append(query, bson.E{"_id", x}) } } if moveGroup != "" { query = append(query, bson.E{"move_group", moveGroup}) } ctx, cancel := context.WithTimeout(context.Background(), findTimeout*time.Second) defer cancel() val, err := c.db3.Database(database).Collection(c.table).Distinct(ctx, fieldName, query) if err != nil { if errors.Is(err, mongo.ErrNoDocuments) { return []string{}, nil } return []string{}, err } ret := make([]string, 0) for _, v := range val { s, _ := v.(string) if s != "" { ret = append(ret, s) } } return ret, nil } // CollectMongo method will take one of account/device/moveGroup and an optional projection to // query Mongo and return a list of mongo.Devices func (c *Client) CollectMongo(account, device, moveGroup string, projection bson.D) (mongoDeviceList []tools.Device, err error) { if c.table == "" { err = errors.New("CollectMongo/no collection selected") return } query := bson.D{} if c.hideDeleted { query = append(query, DeletedNETrue) } if account != "" { x, err := strconv.Atoi(account) if err == nil { query = append(query, bson.E{"account.number", x}) } } if device != "" { x, err := strconv.Atoi(device) if err == nil { query = append(query, bson.E{"_id", x}) } } if moveGroup != "" { query = append(query, bson.E{"move_group", moveGroup}) } opts := new(options.FindOptions) if len(projection) > 0 { opts = options.Find().SetProjection(projection) } ctx, cancel := context.WithTimeout(context.Background(), findTimeout*time.Second) defer cancel() cursor, err := c.db3.Database(database).Collection(c.table).Find(ctx, query, opts) if err != nil { return } mongoDeviceList = make([]tools.Device, 0) err = cursor.All(ctx, &mongoDeviceList) return } // CollectMongoCustom method will query mongo similar to the CollectFromMongo method, however, this method will // pass through bson.D types. func (c Client) CollectMongoCustom(collection string, query bson.D, projection bson.D) (mongoDeviceList []tools.Device, err error) { opts := new(options.FindOptions) if len(projection) > 0 { opts = options.Find().SetProjection(projection) } ctx, cancel := context.WithTimeout(context.Background(), findTimeout*time.Second) defer cancel() cursor, err := c.db3.Database(database).Collection(collection).Find(ctx, query, opts) if err != nil { return } mongoDeviceList = make([]tools.Device, 0) err = cursor.All(ctx, &mongoDeviceList) return } // CollectMongoContainers method will query mongo similar to the CollectFromMongo method, however, this method will // pass through bson.D types. func (c Client) CollectMongoContainers(collection string, query bson.D, projection bson.D) (containers []Container, err error) { opts := new(options.FindOptions) if len(projection) > 0 { opts = options.Find().SetProjection(projection) } ctx, cancel := context.WithTimeout(context.Background(), findTimeout*time.Second) defer cancel() cursor, err := c.db3.Database(database).Collection(collection).Find(ctx, query, opts) if err != nil { return } containers = make([]Container, 0) err = cursor.All(ctx, &containers) return } // FindFilter method will query mongo similar to the CollectFromMongo method, however, this method will // pass through a mongodbAPI.FindQuery type. func (c Client) FindFilter(query bson.D) (containers []EBIFilters, err error) { if c.table == "" { err = errors.New("no collection selected") return } if c.migrationMetadata.FiltersCollection == "" { err = errors.New("filters_collection not set") return } ctx, cancel := context.WithTimeout(context.Background(), findTimeout*time.Second) defer cancel() cursor, err := c.db3.Database(database).Collection(c.migrationMetadata.FiltersCollection).Find(ctx, query) if err != nil { return } containers = make([]EBIFilters, 0) err = cursor.All(ctx, &containers) return } // CollectNetDevices method will query mongo similar to the CollectNetDevices method, however, this method will // pass through a mongodbAPI.FindQuery type. func (c Client) CollectNetDevices(find *mongodbAPI.FindQuery) (mongoDeviceList []NetDevice, err error) { if c.table == "" { err = errors.New("no collection selected") return } resp, err := c.db.Find(find, c.table) if err != nil { return } jd := json.NewDecoder(strings.NewReader(string(resp))) payload := new(findNetDevice) if err = jd.Decode(payload); err != nil { return } mongoDeviceList = payload.Result return } // CollectMigratedEnvironment method will query mongo to see if environment is already migrated func (c *Client) CollectMigratedEnvironment(environment string) ([]MigratedEnvironment, error) { ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() cursor, err := c.db3.Database(database).Collection("environmentsMigrated").Find(ctx, bson.D{{"_id", environment}}) if err != nil { return []MigratedEnvironment{}, err } result := make([]MigratedEnvironment, 0) err = cursor.All(ctx, &result) return result, err } // CollectExcludedMigrationVlans method will query mongo for the list of vlans to be // excluded in the DCX /migrate_vlans method func (c *Client) CollectExcludedMigrationVlans() ([]ZoneVlan, error) { ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) defer cancel() cursor, err := c.db3.Database(database).Collection("vlanTranslationExcluded").Find(ctx, bson.D{}) if err != nil { return []ZoneVlan{}, err } result := make([]ZoneVlan, 0) err = cursor.All(ctx, &result) return result, err } // CollectProcessMaps method will query mongo for the mapping logic used to determine the // process.os_type of a given device. func (c *Client) CollectProcessMaps(qry bson.D) (processMaps []ProcessMapDB, err error) { ctx, cancel := context.WithTimeout(context.Background(), findTimeout*time.Second) defer cancel() opts := options.Find().SetSort(bson.D{{"order", 1}}) cursor, err := c.db3.Database(database).Collection("processMaps").Find(ctx, qry, opts) if err != nil { return } processMaps = make([]ProcessMapDB, 0) err = cursor.All(ctx, &processMaps) return } // CollectZoneMap method will pull list of existing source -> dst zone relationships func (c *Client) CollectZoneMap(sourceDC string) ([]ZoneMap, error) { ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) defer cancel() cursor, err := c.db3.Database(database).Collection(fmt.Sprintf("%sZoneMapping", sourceDC)).Find(ctx, bson.D{}) if err != nil { return []ZoneMap{}, err } result := make([]ZoneMap, 0) err = cursor.All(ctx, &result) return result, err } // CollectVlanTranslations method will query the vlanTranslations DB func (c Client) CollectVlanTranslations(query bson.M) ([]dcxAutohost.EnvironmentMigrationVlan, error) { ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() cursor := c.db3.Database(database).Collection("vlanTranslations").FindOne(ctx, query) type req struct { ID string `json:"_id" bson:"_id"` Xlates []dcxAutohost.EnvironmentMigrationVlan `json:"vlan_translations" bson:"vlan_translations"` } var resp req err := cursor.Decode(&resp) return resp.Xlates, err } // CollectDevicePortMaps method will query the portMaps DB func (c Client) CollectDevicePortMaps(query bson.M) ([]*dcxAutohost.DevicePortMap, error) { ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() cursor, err := c.db3.Database(database).Collection("portMaps").Find(ctx, query) if err != nil { return nil, err } type req struct { ID int `json:"_id" bson:"_id"` PortMaps dcxAutohost.DevicePortMap `json:"port_maps" bson:"port_maps"` } var payload []req if err = cursor.All(ctx, &payload); err != nil { return nil, err } resp := make([]*dcxAutohost.DevicePortMap, 0) //nolint:exportloopref for _, item := range payload { resp = append(resp, &item.PortMaps) } return resp, err } // SyncDB will mainly compare the EBI device data against the mongo collection and look for // devices that do not exist in the migrationDB and remove them. This method can also be used to clean up // any additional items that we deem to be not relevant to a DC migration func (c Client) SyncDB(dDw []int, dMgComplete []tools.Device, col string) (resp []journal.Changes, err error) { for _, device := range dMgComplete { moveGroup := "Unassigned" if device.MoveGroup != "" { moveGroup = device.MoveGroup } exist := false for _, devID := range dDw { if devID == device.ID { exist = true } } // If the device ID does not exist after iterating through the // data-warehouse response, delete the device from mongo and alert. if !exist && !device.Deleted { err := c.UpdateOne( col, bson.D{{"_id", device.ID}}, bson.D{{"$set", bson.D{{"deleted", true}, {"device.status", "Deleted"}}}}, false, ) if err != nil { continue } resp = append(resp, journal.Changes{ EventType: "DeviceDeleted", AccountID: device.Account.Number, ID: strconv.Itoa(device.ID), Reason: fmt.Sprintf("%s=%s", moveGroup, device.Device.Status), TeamName: device.Account.TeamName, }) } if exist && device.Deleted { err := c.UpdateOne( col, bson.D{{"_id", device.ID}}, bson.D{{"$set", bson.D{{"deleted", false}}}}, false, ) if err != nil { continue } resp = append(resp, journal.Changes{ EventType: "DeviceReAdded", AccountID: device.Account.Number, ID: strconv.Itoa(device.ID), Reason: fmt.Sprintf("%s=%s", moveGroup, device.Device.Status), TeamName: device.Account.TeamName, }) } } return }