// 2017 Rackspace, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"): you may // not use this file except in compliance with the License. You may obtain // a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the // License for the specific language governing permissions and limitations // under the License. package ctk import ( "fmt" "math" "strconv" "strings" "github.rackspace.com/DCM/tools" "go.mongodb.org/mongo-driver/bson" "github.com/sirupsen/logrus" "github.rackspace.com/DCM/migration/pkg/collection" "github.rackspace.com/DCM/migration/pkg/tools/journal" "github.rackspace.com/gorax/tools/util" ) const ( routines = 5 // LimitCTKQuery is the number of items to process through a single // CTK call before chunking into multiple calls. LimitCTKQuery = 240 ) // UpdateIPsFromCTK method will take an account or device, and poll CTK API for IP address information for each // device that is matched. func (c *Client) UpdateIPsFromCTK(account, device, updateOnly string) (resp collection.UpdateResponse, err error) { dl, err := c.db.CollectMongo(account, device, "", bson.D{}) if err != nil { return } for _, item := range dl { if item.Device.PrimaryIP != "" { continue } r, err := c.ctk.ComputerQuery(item.ID) if err != nil { resp.Missed = append(resp.Missed, collection.Missed{Item: item.ID, Details: err.Error()}) continue } query := bson.D{{"_id", item.ID}} update := bson.D{{"$set", bson.D{{"device.primary_ip", r.PrimaryIP}}}} if uerr := c.db.UpdateOne("dfw1", query, update, true); uerr != nil { resp.Missed = append(resp.Missed, collection.Missed{Item: item.ID, Details: uerr.Error()}) continue } resp.Updated++ } return } // UpdateMigrationDB method will kick off a series of go routines to post device information to the migration // database using data collected from GDCI EBI. func (c *Client) UpdateMigrationDB(devices []int, table string) (resp collection.UpdateResponse, err error) { dl := make([]Device, 0) // Pull MRR from Datamart. If we encounter an error, we will still continue processing, // just log that an error occurred. mrrMap, err := c.dMart.DMartMRR(strings.ToUpper(c.db.CollectionDC())) if err != nil { logrus.Errorf("error collecting MRR from DMART: %s", err) } logrus.Infof("collecting for all devices %d", len(devices)) ctkCnt := int(math.Ceil(float64(len(devices)) / float64(LimitCTKQuery))) // Break the device list into the CTK query size for _, devChunk := range ChunkString(devices, ctkCnt) { ctkResp, err := c.CollectFromCTK(devChunk, mrrMap) if err != nil { logrus.Errorf("migration:ctk:UpdateMigrationDB::CollectFromCTK:%s", err) continue } logrus.Infof("collected devices %d", len(ctkResp)) dl = append(dl, ctkResp...) } chMissed := make(chan []collection.Missed) chInt := make(chan int) chnk := Chunk(dl, routines) defer close(chMissed) defer close(chInt) for _, item := range chnk { go runCTKImportRoutine(c, item, chMissed, chInt, table) } for range chnk { resp.Missed = append(resp.Missed, <-chMissed...) resp.Updated += <-chInt } return } // UpdateMigrationDBForDC method will kick off a series of go routines to post device information to the migration // database using data collected from GDCI EBI. func (c *Client) UpdateMigrationDBForDC(ctkDevices []Device, table string) (resp collection.UpdateResponse, err error) { chMissed := make(chan []collection.Missed) chInt := make(chan int) chnk := Chunk(ctkDevices, routines) defer close(chMissed) defer close(chInt) for _, item := range chnk { go runCTKImportRoutine(c, item, chMissed, chInt, table) } for range chnk { resp.Missed = append(resp.Missed, <-chMissed...) resp.Updated += <-chInt } return } // runEBIEMRoutine function is the internal function to kick off routines for UpdateMigrationDB. func runCTKImportRoutine(c *Client, dl []Device, chMissed chan<- []collection.Missed, chInt chan<- int, table string) { var resp collection.UpdateResponse for _, item := range dl { query := bson.D{{"_id", item.ID}} update := bson.D{{ "$set", bson.D{ {"san", item.SAN}, {"has_das", item.DAS}, {"has_mbu", item.MBU}, {"has_sharepoint", item.SharePoint}, {"has_rackconnect", item.RackConnect}, {"has_mssql_support", item.MSSQLSupport}, {"has_mysql_support", item.MySQLSupport}, {"has_oracle_support", item.OracleSupport}, {"has_skype", item.Skype}, {"has_doubletake", item.DoubleTake}, {"has_dex", item.DEX}, {"rcv2", item.RCv2}, {"rcv3", item.RCv3}, {"account", map[string]interface{}{ "geographic_location": item.Account.GeographicLocation, "lead_tech": item.Account.LeadTech, "manager": item.Account.AM, "bdc": item.Account.BDC, "mrr": item.Account.CMRR, "name": item.Account.Name, "number": item.Account.Number, "status": item.Account.Status, "team_name": item.Account.TeamName, "type": item.Account.Type, "primary_contact": item.Account.PrimaryContact, }}, {"device", map[string]interface{}{ "cmrr": item.Device.CMRR, "container_component_configuration_type_name": item.Device.ComponentConfigType, "hostname": item.Device.HostName, "make_model": item.Device.MakeModel, "online_status": item.Device.OnlineStatus, "os": item.Device.OS, "datacenter": item.Device.DataCenter, "os_name": item.Device.OSName, "status": item.Device.Status, "primary_ip": item.Device.PrimaryIP, "type": item.Device.Type, "sla": item.Device.SLA, "customer_managed": item.Device.CustomerManaged, }}, {"dp_device", item.DPDevice}, {"dp_account", item.DPAccount}, {"deleted", item.Device.Status == "Computer No Longer Active"}, {"icon_url", item.IconURL}, {"_table", table}, {"_db", collection.DBName}, }, }} if uerr := c.db.UpdateOne(table, query, update, true); uerr != nil { resp.Missed = append(resp.Missed, collection.Missed{Item: item.ID, Details: fmt.Sprintf("Mongo UpdateOne %v", uerr.Error())}) continue } resp.Updated++ } chMissed <- resp.Missed chInt <- resp.Updated } // CleanFromMongo will compare the CTK device data returned from CORE against the mongo collection and look for // devices that do not exist in the migrationDB and remove them. This method can also be used to clean up // any additional items that we deem to be not relevant to a DC migration func (c Client) CleanFromMongo(ctkDevice []Device, mongoDevice []tools.Device, table string, variance int) (resp collection.UpdateResponse, err error) { ok := false if ok, err = util.WithinVariance(len(ctkDevice), len(mongoDevice), variance); !ok { err = nil return } for _, mgo := range mongoDevice { exist := false for _, e := range ctkDevice { if e.ID == mgo.ID { exist = true } } // If the device ID does not exist after iterating through the // data-warehouse response, delete the device from mongo and alert. if !exist && !mgo.Deleted { err := c.db.UpdateOne( table, bson.D{{"_id", mgo.ID}}, bson.D{{"$set", bson.D{{"deleted", true}, {"device.status", "Deleted"}}}}, false) if err != nil { resp.Missed = append(resp.Missed, collection.Missed{Item: mgo.ID, Details: err.Error()}) continue } resp.Updated++ resp.Changes = append(resp.Changes, journal.Changes{ EventType: "DeviceDeleted", AccountID: mgo.Account.Number, ID: strconv.Itoa(mgo.ID), Reason: fmt.Sprintf("%s=%s", mgo.MoveGroup, mgo.Device.Status), TeamName: mgo.Account.TeamName, }) } if exist && mgo.Deleted { err := c.db.UpdateOne( table, bson.D{{"_id", mgo.ID}}, bson.D{{"$set", bson.D{{"deleted", false}}}}, false) if err != nil { resp.Missed = append(resp.Missed, collection.Missed{Item: mgo.ID, Details: err.Error()}) continue } resp.Updated++ resp.Changes = append(resp.Changes, journal.Changes{ EventType: "DeviceReAdded", AccountID: mgo.Account.Number, ID: strconv.Itoa(mgo.ID), Reason: fmt.Sprintf("%s=%s", mgo.MoveGroup, mgo.Device.Status), TeamName: mgo.Account.TeamName, }) } } return }