// 2017 Rackspace, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"): you may // not use this file except in compliance with the License. You may obtain // a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the // License for the specific language governing permissions and limitations // under the License. package mongo import ( "context" "errors" "fmt" "log" "strconv" "time" "github.rackspace.com/gorax/clients/fenetsec" "github.rackspace.com/DCM/discover" "github.rackspace.com/DCM/migration/pkg/tools/journal" "github.rackspace.com/DCM/tools" "github.rackspace.com/gorax/clients/dcxAutohost" "go.mongodb.org/mongo-driver/bson" goMongo "go.mongodb.org/mongo-driver/mongo" "go.mongodb.org/mongo-driver/mongo/options" ) const ( prepDiscover = "discover" prepFinalize = "finalize" prepCopySegments = "copy_segments" prepEnvTrueUp = "env_true_up" prepEnvTranslations = "translations" validateBackup = "backups" validatePortConfigs = "migration_port_configs" statusCollection = "status" // StatusAuditColoTrunks defines the audits field name in the status collection StatusAuditColoTrunks = "colo_trunks" // StatusAuditMigrationIPs defines the audits field name in the status collection StatusAuditMigrationIPs = "migration_ips" // StatusAuditMissingDB defines the audits field name in the status collection StatusAuditMissingDB = "missing_db" // StatusAuditMissingEnv defines the audits field name in the status collection StatusAuditMissingEnv = "missing_env" // StatusAuditNetworkArchitect defines the audits field name in the status collection StatusAuditNetworkArchitect = "network_architect" // StatusAuditPortAssignments defines the audits field name in the status collection StatusAuditPortAssignments = "port_assignments" // StatusAuditComplexServers defines the audits field name in the status collection // StatusAuditComplexServers = "complex_servers" ) type ( statusDoc struct { ID string `bson:"_id"` Audits statusAudit `bson:"audits"` Prep statusPrep `bson:"prep"` Validate validate `bson:"validate,omitempty"` } statusAudit struct { ColoTrunks statusAuditInfo `bson:"colo_trunks"` //ComplexServers statusAuditInfo `bson:"complex_servers"` MissingDB statusAuditInfo `bson:"missing_db"` MissingEnv statusAuditInfo `bson:"missing_env"` PortAssignments statusAuditInfo `bson:"port_assignments"` NetworkArchitect statusAuditInfo `bson:"network_architect"` } statusAuditInfo struct { ManualOverride bool `bson:"manual_override"` Alerts []journal.Changes `bson:"alerts"` Runs []statusAuditSummary `bson:"runs"` } statusAuditSummary struct { AlertCount int `bson:"alert_count"` Timestamp int64 `bson:"timestamp"` } statusPrep struct { CopySegments []copySegmentsPrep `json:"copy_segments,omitempty" bson:"copy_segments,omitempty"` Discover []discoverPrep `json:"discover,omitempty" bson:"discover,omitempty"` Finalize []finalizePrep `json:"finalize,omitempty" bson:"finalize,omitempty"` TrueUp []envTrueUpPrep `json:"env_true_up,omitempty" bson:"env_true_up,omitempty"` Translations []envXlatesPrep `json:"translations,omitempty" bson:"translations,omitempty"` MigrationIPs statusAuditInfo `json:"migration_ips" bson:"migration_ips"` } validate struct { Portconfigs []validateDevice `json:"migration_port_configs,omitempty" bson:"migration_port_configs,omitempty"` Backups []validateBackups `json:"backups,omitempty" bson:"backups,omitempty"` } copySegmentsPrep struct { ID int `json:"_id" bson:"_id"` Deleted bool `json:"deleted" bson:"deleted"` Ignore bool `json:"ignore,omitempty" bson:"ignore,omitempty"` Results []StatusCopySegments `json:"results,omitempty" bson:"results,omitempty"` } envTrueUpPrep struct { Deleted bool `json:"deleted" bson:"deleted"` Number string `json:"number" bson:"number"` Ignore bool `json:"ignore,omitempty" bson:"ignore,omitempty"` Results []StatusEnvTrueUp `json:"results,omitempty" bson:"results,omitempty"` } envXlatesPrep struct { Deleted bool `json:"deleted" bson:"deleted"` Number string `json:"number" bson:"number"` Ignore bool `json:"ignore,omitempty" bson:"ignore,omitempty"` Results []StatusEnvXlates `json:"results,omitempty" bson:"results,omitempty"` } discoverPrep struct { ID int `json:"_id" bson:"_id"` // account number Deleted bool `json:"deleted" bson:"deleted"` Ignore bool `json:"ignore,omitempty" bson:"ignore,omitempty"` Results []StatusDiscover `json:"results,omitempty" bson:"results,omitempty"` } finalizePrep struct { ID int `json:"_id" bson:"_id"` // device number Deleted bool `json:"deleted" bson:"deleted"` Ignore bool `json:"ignore,omitempty" bson:"ignore,omitempty"` Results []StatusFinalize `json:"results,omitempty" bson:"results,omitempty"` } validateDevice struct { ID int `json:"_id" bson:"_id"` Deleted bool `json:"deleted" bson:"deleted"` Ignore bool `json:"ignore,omitempty" bson:"ignore,omitempty"` Results []StatusValidateResult `json:"results,omitempty" bson:"results,omitempty"` } validateBackups struct { ID int `json:"_id" bson:"_id"` Deleted bool `json:"deleted" bson:"deleted"` Ignore bool `json:"ignore,omitempty" bson:"ignore,omitempty"` Results []StatusValidateBackups `json:"results,omitempty" bson:"results,omitempty"` } // StatusCopySegments defines a POST payload for the /prep/copy_segments method StatusCopySegments struct { ID int `json:"_id" bson:"-"` Username string `json:"username" bson:"username"` Timestamp int64 `json:"timestamp" bson:"timestamp"` Error string `json:"error" bson:"error"` } // StatusDiscover defines a POST payload for the /prep/discover method StatusDiscover struct { ID int `json:"_id" bson:"-"` Username string `json:"username" bson:"username"` Timestamp int64 `json:"timestamp" bson:"timestamp"` Result discover.Result `json:"result" bson:"result"` } // StatusFinalize defines a POST payload for the /prep/finalize method StatusFinalize struct { ID int `json:"_id" bson:"-"` Username string `json:"username" bson:"username"` Timestamp int64 `json:"timestamp" bson:"timestamp"` JobStatus dcxAutohost.JobStatus `json:"job_status" bson:"job_status"` } // StatusEnvTrueUp defines a POST payload for the /prep/environment_true_up method StatusEnvTrueUp struct { Number string `json:"number" bson:"-"` Username string `json:"username" bson:"username"` Timestamp int64 `json:"timestamp" bson:"timestamp"` Result dcxAutohost.EnvironmentAutoDiscover `json:"result" bson:"result"` Error string `json:"error" bson:"error"` } // StatusEnvXlates defines a POST payload for the /prep/environment_translations method StatusEnvXlates struct { Number string `json:"number" bson:"-"` Username string `json:"username" bson:"username"` Timestamp int64 `json:"timestamp" bson:"timestamp"` Result []dcxAutohost.EnvironmentMigrationVlan `json:"result" bson:"result"` Error string `json:"error" bson:"error"` } // StatusValidateBackups defines a POST payload for the /prep/backups method StatusValidateBackups struct { ID int `json:"_id" bson:"-"` Username string `json:"username" bson:"username"` Timestamp int64 `json:"timestamp" bson:"timestamp"` Result fenetsec.DeviceEvent `json:"result" bson:"result"` } // StatusValidateResult defines a validate result StatusValidateResult struct { Error string `json:"error" bson:"error"` Result []journal.Changes `json:"result" bson:"result"` Timestamp int64 `json:"timestamp" bson:"timestamp"` Username string `json:"username" bson:"username"` } ) func (c *Client) findAuditResults(movegroup, audit string) (*statusDoc, error) { ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() field := fmt.Sprintf("audits.%s.alerts", audit) if audit == StatusAuditMigrationIPs { field = fmt.Sprintf("prep.%s.alerts", audit) } audits := new(statusDoc) err := c.db3.Database(database).Collection(statusCollection).FindOne( ctx, bson.D{{"_id", movegroup}}, options.FindOne().SetProjection(bson.D{{field, 1}}), ).Decode(audits) if err == goMongo.ErrNoDocuments { err = nil } return audits, err } func (c *Client) findAccountIndex(movegroup string, account int) (*int, error) { ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() s := new(statusDoc) err := c.db3.Database(database).Collection(statusCollection).FindOne( ctx, bson.D{{"_id", movegroup}}, options.FindOne().SetProjection(bson.D{ {fmt.Sprintf("prep.%s", prepDiscover), 1}, }), ).Decode(s) if err != nil { return nil, err } if s.Prep.Discover == nil { return nil, nil } for i, x := range s.Prep.Discover { if x.ID == account { return &i, nil } } return nil, nil } func (c *Client) findDeviceIndex(movegroup string, device int, prepType string) (*int, error) { ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() s := new(statusDoc) err := c.db3.Database(database).Collection(statusCollection).FindOne( ctx, bson.D{{"_id", movegroup}}, options.FindOne().SetProjection(bson.D{ {fmt.Sprintf("prep.%s", prepCopySegments), 1}, {fmt.Sprintf("prep.%s", prepFinalize), 1}, {fmt.Sprintf("validate.%s", validateBackup), 1}, {fmt.Sprintf("validate.%s", validatePortConfigs), 1}, }), ).Decode(s) if err != nil { return nil, err } switch prepType { case prepCopySegments: if s.Prep.CopySegments == nil { return nil, nil } for i, x := range s.Prep.CopySegments { if x.ID == device { return &i, nil } } return nil, nil case prepFinalize: if s.Prep.Finalize == nil { return nil, nil } for i, x := range s.Prep.Finalize { if x.ID == device { return &i, nil } } return nil, nil case validateBackup: if s.Validate.Backups == nil { return nil, nil } for i, x := range s.Validate.Backups { if x.ID == device { return &i, nil } } return nil, nil case validatePortConfigs: if s.Validate.Portconfigs == nil { return nil, nil } for i, x := range s.Validate.Portconfigs { if x.ID == device { return &i, nil } } return nil, nil } return nil, errors.New("unknown prep type") } func (c *Client) findEnvIndex(movegroup, envNumber, prepType string) (*int, error) { ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() s := new(statusDoc) err := c.db3.Database(database).Collection(statusCollection).FindOne( ctx, bson.D{{"_id", movegroup}}, options.FindOne().SetProjection(bson.D{{fmt.Sprintf("prep.%s", prepType), 1}}), ).Decode(s) if err != nil { return nil, err } switch prepType { case prepEnvTrueUp: if s.Prep.TrueUp == nil { return nil, nil } for i, x := range s.Prep.TrueUp { if x.Number == envNumber { return &i, nil } } return nil, nil case prepEnvTranslations: if s.Prep.Translations == nil { return nil, nil } for i, x := range s.Prep.Translations { if x.Number == envNumber { return &i, nil } } return nil, nil } return nil, errors.New("unknown prep type") } // UpdateStatus is a method that allows audits to update their status results in the DB func (c *Client) UpdateStatus(movegroup, audit string, results []journal.Changes) error { if audit == StatusAuditMissingDB { return c.updateStatusByCollection(movegroup, audit, results) } return c.updateStatusByMG(movegroup, audit, results) } // UpdateStatusList is a method that allows audits to update their status results in the DB func (c *Client) UpdateStatusList(movegroups []string, audit string, results []journal.Changes) error { if len(movegroups) == 1 { return c.updateStatusByMG(movegroups[0], audit, results) } m, err := tools.New(c.db3, nil).CollectMetadataByMG(movegroups, true) if err != nil { return err } for _, mg := range m { mgResults := make([]journal.Changes, 0) for _, r := range results { devID, err := strconv.Atoi(r.ID) if err != nil { continue } if deviceInList(devID, mg.Devices) { mgResults = append(mgResults, r) } } if err = c.updateStatusByMG(mg.ID, audit, mgResults); err != nil { return err } } return nil } // loop through all active move groups matching results to environments in MG func (c *Client) updateStatusByCollection(collection, audit string, results []journal.Changes) error { m, err := tools.New(c.db3, nil).CollectMetadata(&collection, true) if err != nil { return err } for _, mg := range m { mgResults := make([]journal.Changes, 0) for _, r := range results { if envInList(r.Current, mg.Environments) { mgResults = append(mgResults, r) } } if err = c.updateStatusByMG(mg.ID, audit, mgResults); err != nil { return err } } return nil } func (c *Client) updateStatusByMG(movegroup, audit string, results []journal.Changes) error { updates := make([]journal.Changes, 0) audits, err := c.findAuditResults(movegroup, audit) if err != nil { return err } fieldPrefix := "audits" current := audits.Audits.ColoTrunks.Alerts switch audit { //case StatusAuditComplexServers: // current = audits.Audits.ComplexServers.Alerts case StatusAuditMigrationIPs: current = audits.Prep.MigrationIPs.Alerts fieldPrefix = "prep" case StatusAuditMissingDB: current = audits.Audits.MissingDB.Alerts case StatusAuditMissingEnv: current = audits.Audits.MissingEnv.Alerts case StatusAuditNetworkArchitect: current = audits.Audits.NetworkArchitect.Alerts case StatusAuditPortAssignments: current = audits.Audits.PortAssignments.Alerts } // save any manually ignored/ack'd rows for _, a := range current { if a.Ignore { updates = append(updates, a) } } for _, r := range results { exists := false for _, a := range updates { if a.ID == r.ID && a.Reason == r.Reason && a.Ignore { exists = true break } } if !exists { updates = append(updates, r) } } sum := statusAuditSummary{ AlertCount: len(results), Timestamp: time.Now().Unix(), } return c.UpdateOne( statusCollection, bson.D{{"_id", movegroup}}, bson.D{ {"$set", bson.D{{fmt.Sprintf("%s.%s.alerts", fieldPrefix, audit), updates}}}, {"$push", bson.D{{fmt.Sprintf("%s.%s.runs", fieldPrefix, audit), sum}}}, }, true, ) } // UpdateStatusBackups updates the DB with the latest validate.backups results func (c *Client) UpdateStatusBackups(collection string, status []StatusValidateBackups) error { metadata, err := tools.New(c.db3, nil).CollectMetadata(&collection, true) if err != nil { return err } for _, s := range status { if s.Timestamp == 0 { s.Timestamp = time.Now().Unix() } movegroup := "" for _, m := range metadata { for _, d := range m.Devices { if s.ID == d { movegroup = m.ID break } } if movegroup != "" { break } } if movegroup == "" { log.Printf("UpdateStatusBackups: no move group found for %d", s.ID) continue } index, err := c.findDeviceIndex(movegroup, s.ID, validateBackup) if err != nil && err != goMongo.ErrNoDocuments { return err } update := bson.D{{ "$push", bson.D{{ fmt.Sprintf("validate.%s", validateBackup), validateBackups{ID: s.ID, Results: []StatusValidateBackups{s}}, }}, }} if index != nil { update = bson.D{{ "$push", bson.D{{fmt.Sprintf("validate.%s.%d.results", validateBackup, *index), s}}, }} } err = c.UpdateOne( statusCollection, bson.D{{"_id", movegroup}}, update, true, ) if err != nil { return err } } return nil } // UpdateStatusDiscover updates the DB with the latest discover results func (c *Client) UpdateStatusDiscover(movegroup string, status StatusDiscover) error { if status.Timestamp == 0 { status.Timestamp = time.Now().Unix() } index, err := c.findAccountIndex(movegroup, status.ID) if err != nil && err != goMongo.ErrNoDocuments { return err } update := bson.D{{ "$push", bson.D{{ fmt.Sprintf("prep.%s", prepDiscover), discoverPrep{ID: status.ID, Results: []StatusDiscover{status}}, }}, }} if index != nil { update = bson.D{{ "$push", bson.D{{fmt.Sprintf("prep.%s.%d.results", prepDiscover, *index), status}}, }} } err = c.UpdateOne( statusCollection, bson.D{{"_id", movegroup}}, update, true, ) if err != nil { return err } return nil } // UpdateStatusCopySegments updates the DB with the latest copy_segments results func (c *Client) UpdateStatusCopySegments(collection string, status []StatusCopySegments) error { metadata, err := tools.New(c.db3, nil).CollectMetadata(&collection, true) if err != nil { return err } for _, s := range status { if s.Timestamp == 0 { s.Timestamp = time.Now().Unix() } movegroup := "" for _, m := range metadata { for _, d := range m.Devices { if s.ID == d { movegroup = m.ID break } } if movegroup != "" { break } } if movegroup == "" { log.Printf("UpdateStatusCopySegments: no move group found for %d", s.ID) continue } index, err := c.findDeviceIndex(movegroup, s.ID, prepCopySegments) if err != nil && err != goMongo.ErrNoDocuments { return err } update := bson.D{{ "$push", bson.D{{ fmt.Sprintf("prep.%s", prepCopySegments), copySegmentsPrep{ID: s.ID, Results: []StatusCopySegments{s}}, }}, }} if index != nil { update = bson.D{{ "$push", bson.D{{fmt.Sprintf("prep.%s.%d.results", prepCopySegments, *index), s}}, }} } err = c.UpdateOne( statusCollection, bson.D{{"_id", movegroup}}, update, true, ) if err != nil { return err } } return nil } // UpdateStatusFinalize updates the DB with the latest finalize results func (c *Client) UpdateStatusFinalize(collection string, status []StatusFinalize) error { metadata, err := tools.New(c.db3, nil).CollectMetadata(&collection, true) if err != nil { return err } for _, s := range status { if s.Timestamp == 0 { s.Timestamp = time.Now().Unix() } movegroup := "" for _, m := range metadata { for _, d := range m.Devices { if s.ID == d { movegroup = m.ID break } } if movegroup != "" { break } } if movegroup == "" { log.Printf("UpdateStatusFinalize: no move group found for %d", s.ID) continue } index, err := c.findDeviceIndex(movegroup, s.ID, prepFinalize) if err != nil && err != goMongo.ErrNoDocuments { return err } update := bson.D{{ "$push", bson.D{{ fmt.Sprintf("prep.%s", prepFinalize), finalizePrep{ID: s.ID, Results: []StatusFinalize{s}}, }}, }} if index != nil { update = bson.D{{ "$push", bson.D{{fmt.Sprintf("prep.%s.%d.results", prepFinalize, *index), s}}, }} } err = c.UpdateOne( statusCollection, bson.D{{"_id", movegroup}}, update, true, ) if err != nil { return err } } return nil } // UpdateStatusTrueUp updates the DB with the latest true-up results func (c *Client) UpdateStatusTrueUp(movegroup string, status []StatusEnvTrueUp) error { for _, s := range status { if s.Timestamp == 0 { s.Timestamp = time.Now().Unix() } index, err := c.findEnvIndex(movegroup, s.Number, prepEnvTrueUp) if err != nil && err != goMongo.ErrNoDocuments { return err } update := bson.D{{ "$push", bson.D{{ fmt.Sprintf("prep.%s", prepEnvTrueUp), envTrueUpPrep{Number: s.Number, Results: []StatusEnvTrueUp{s}}, }}, }} if index != nil { update = bson.D{{ "$push", bson.D{{fmt.Sprintf("prep.%s.%d.results", prepEnvTrueUp, *index), s}}, }} } err = c.UpdateOne( statusCollection, bson.D{{"_id", movegroup}}, update, true, ) if err != nil { return err } } return nil } // UpdateStatusXlates updates the DB with the latest true-up results func (c *Client) UpdateStatusXlates(movegroup string, status []StatusEnvXlates) error { for _, s := range status { if s.Timestamp == 0 { s.Timestamp = time.Now().Unix() } index, err := c.findEnvIndex(movegroup, s.Number, prepEnvTranslations) if err != nil && err != goMongo.ErrNoDocuments { return err } update := bson.D{{ "$push", bson.D{{ fmt.Sprintf("prep.%s", prepEnvTranslations), envXlatesPrep{Number: s.Number, Results: []StatusEnvXlates{s}}, }}, }} if index != nil { update = bson.D{{ "$push", bson.D{{fmt.Sprintf("prep.%s.%d.results", prepEnvTranslations, *index), s}}, }} } err = c.UpdateOne( statusCollection, bson.D{{"_id", movegroup}}, update, true, ) if err != nil { return err } } return nil } // UpdateStatusValidate updates the DB with the latest validate port_configs results func (c *Client) UpdateStatusValidate(collection string, d string, result StatusValidateResult) error { device, err := strconv.Atoi(d) if err != nil { return err } metadata, err := tools.New(c.db3, nil).CollectMetadata(&collection, true) if err != nil { return err } movegroup := "" for _, m := range metadata { for _, d := range m.Devices { if device == d { movegroup = m.ID break } } if movegroup != "" { break } } if movegroup == "" { return fmt.Errorf("UpdateStatusValidate: no move group found for %d", device) } index, err := c.findDeviceIndex(movegroup, device, validatePortConfigs) if err != nil && err != goMongo.ErrNoDocuments { return err } update := bson.D{{ "$push", bson.D{{ fmt.Sprintf("validate.%s", validatePortConfigs), validateDevice{ID: device, Results: []StatusValidateResult{result}}, }}, }} if index != nil { update = bson.D{{ "$push", bson.D{{fmt.Sprintf("validate.%s.%d.results", validatePortConfigs, *index), result}}, }} } err = c.UpdateOne( statusCollection, bson.D{{"_id", movegroup}}, update, true, ) if err != nil { return err } return nil } func deviceInList(device int, devices []int) bool { for _, d := range devices { if d == device { return true } } return false } func envInList(env string, envs []string) bool { for _, e := range envs { if e == env { return true } } return false }