// 2017 Rackspace, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"): you may // not use this file except in compliance with the License. You may obtain // a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the // License for the specific language governing permissions and limitations // under the License. // Package ebi extends the EBI-ODS CORE database collection into the Migration Database. package ebi import ( "math" "net/http" "strconv" "strings" "go.mongodb.org/mongo-driver/bson/primitive" "github.rackspace.com/DCM/migration/pkg/collection/mongo" "github.com/sirupsen/logrus" "github.rackspace.com/DCM/migration/pkg/collection" "github.rackspace.com/gorax/clients/raxSQL" "github.rackspace.com/gorax/passkey" ) const ( // routines to use for importing of data. Each call will spawn x number of routines // to run the calls to populate the database. routines = 10 iconPrefix = "https://28616ea5aab979dce92c-69a5bc1f8c3c585c4f658970ef961bb7.ssl.cf2.rackcdn.com/" sqlQueryTimeout = 10800 // Time in seconds // LinkedByDetails is a flag used to determine the type of link specified in the linked sub-document LinkedByDetails = "110132-Device-Details" // LinkedByIntegration is a flag used to determine the type of link specified in the linked sub-document LinkedByIntegration = "110339-Flagged-Integration" ) // AlwaysImportModels is a list of device.make_models that we will always import into a collection // that is being filtered as they are unable to be associated to an EM environment, but // are still important to the migration. var AlwaysImportModels = []string{ "Leased Line", "Duo Multi Factor Authentication", "Security Services - Incapsula Cloud Web Application Firewall", "Raritan KVM", "RackConnect Global - Internal", "Rackspace DDoS Mitigation", "Managed Storage", "MBU Backoffice", "Dedicated Aggr Switch", "Switch", } // AlwaysImportAccounts is a list of accounts per-collection that we should override the // environment import method and import all devices for the account in the DC into the // collection. var AlwaysImportAccounts = map[string][]int{ "ord1_01": { // I3 Verticals, LLC. - Devices in CR5 are moving in CR8 migration 132952, }, } type ( // Filters type is a structure returned from the Filters table in mongo Filters struct { primitive.ObjectID `json:"_id,omitempty" bson:"_id,omitempty"` Value string `json:"value" bson:"value"` Type string `json:"type" bson:"type"` } // Client type represents client references to third-party resources required // for the methods within this package. Client struct { sql *raxSQL.Client db *mongo.Client } // Device struct is a representation of a device that we get back // from the GDCI implementation of a EBI-ODS device Device struct { ID int `json:"_id" bson:"_id"` Account struct { Number int `json:"number" bson:"number"` Name string `json:"name" bson:"name"` Type string `json:"type" bson:"type"` LeadTech string `json:"lead_tech" bson:"lead_tech"` Status string `json:"status" bson:"status"` TeamName string `json:"team_name" bson:"team_name"` GeographicLocation string `json:"geographic_location" bson:"geographic_location"` CMRR string `json:"cmrr,omitempty" bson:"cmrr,omitempty"` AM string `json:"account_manager" bson:"account_manager"` BDC string `json:"account_bdc" bson:"account_bdc"` PrimaryContact string `json:"primary_contact" bson:"primary_contact"` } `json:"account" bson:"account"` Device struct { OS string `json:"os" bson:"os"` HostName string `json:"hostname" bson:"hostname"` DataCenter string `json:"datacenter" bson:"datacenter"` Type string `json:"type" bson:"type"` OnlineStatus string `json:"online_status" bson:"online_status"` Status string `json:"status" bson:"status"` PrimaryIP string `json:"primary_ip" bson:"primary_ip"` CMRR string `json:"cmrr" bson:"cmrr"` MakeModel string `json:"make_model" bson:"make_model"` OSName string `json:"os_name,omitempty" bson:"os_name,omitempty"` ComponentConfigType string `json:"container_component_configuration_type_name" bson:"container_component_configuration_type_name"` SLA string `json:"device_sla" bson:"device_sla"` UHeight string `json:"u_height" bson:"u_height"` CustomerManaged bool `json:"customer_managed" bson:"customer_managed"` } `json:"device" bson:"device"` Environment struct { Current Location `json:"current,omitempty"` Migration Location `json:"migration,omitempty"` Number string `json:"number,omitempty"` } `json:"environment,omitempty"` SAN bool `json:"san"` DAS bool `json:"has_das"` SharePoint bool `json:"has_sharepoint"` MBU bool `json:"has_mbu"` DEX bool `json:"has_dex"` DoubleTake bool `json:"has_doubletake"` Skype bool `json:"has_skype"` RackConnect bool `json:"has_rackconnect"` MSSQLSupport bool `json:"has_mssql_support"` MySQLSupport bool `json:"has_mysql_support"` OracleSupport bool `json:"has_oracle_support"` IconURL string `json:"icon_url"` RCv2 bool `json:"rcv2"` RCv3 bool `json:"rcv3"` DPDevice string `json:"datapipe_device"` DPAccount string `json:"datapipe_account"` Linked struct { ID int `json:"_id,omitempty"` AccountNumber int `json:"account_number,omitempty"` AccountName string `json:"account_name,omitempty"` Type string `json:"type,omitempty"` } `json:"linked,omitempty"` } // DeviceMRR represents the data received from DataMart for a // devices monthly revenue. DeviceMRR struct { Account string `json:"account" bson:"account"` Datacenter string `json:"data_center" bson:"data_center"` USD string `json:"usd" bson:"usd"` } // Location type is a representation of the physical assignment of // a device in the datacenter. Location struct { Cabinet string `json:"cabinet,omitempty" bson:"cabinet,omitempty"` Location string `json:"location,omitempty" bson:"location,omitempty"` DataCenter string `json:"data_center" bson:"data_center"` Container string `json:"container" bson:"container"` StartingSpace int `json:"starting_space" bson:"starting_space"` SwitchPorts []string `json:"switch_ports" bson:"switch_ports"` } // SKUData represents the data about a SKU we get back from the // EBI query to the sku_extended_attributes table. SKUData map[string]SKUUnit // SKUUnit represents the details about a given SKU number SKUUnit struct { Name string `json:"name" bson:"name"` Attr []SKUAttr `json:"attribute" bson:"attribute"` } // SKUAttr represents the attribute values for a given // SKU Unit SKUAttr struct { Name string `json:"name" bson:"name"` AttrValue string `json:"value" bson:"value"` Label string `json:"label" bson:"label"` } ) // NewGDCI function will take a pointer to a Request, and pull the client references // from the Context() offered within the default net/http package. func NewGDCI(r *http.Request) (Client, error) { pk := passkey.FromContext(r.Context()) newSQL, err := raxSQL.NewMSSQL( raxSQL.HostEBIGDCI, raxSQL.AuthSQL{ Username: pk.Service.GDCIDataMart.Credential.Username, Password: pk.Service.GDCIDataMart.Credential.Password, }, raxSQL.WithTimeout(sqlQueryTimeout), ) return Client{ sql: newSQL, db: mongo.NewContext(r), }, err } // NewDMART function will take a pointer to a Request, and pull the client references // from the Context() offered within the default net/http package. func NewDMART(r *http.Request) (Client, error) { pk := passkey.FromContext(r.Context()) newSQL, err := raxSQL.NewMSSQL( raxSQL.HostEBIDataMart, raxSQL.AuthWindows{ Username: pk.Service.SSO.Credential.Username, Password: pk.Service.SSO.Credential.Password, Domain: "RACKSPACE", }, raxSQL.WithTimeout(sqlQueryTimeout), ) return Client{ sql: newSQL, db: mongo.NewContext(r), }, err } // Chunk is a small internal function to break a list of Device types into // smaller chunks that we can send to go routines. func Chunk(devList []Device, size int) (chunk [][]Device) { float := float64(len(devList)) / float64(size) chunkSize := int(math.Ceil(float)) for i := 0; i < len(devList); i += chunkSize { end := i + chunkSize if end > len(devList) { end = len(devList) } chunk = append(chunk, devList[i:end]) } return } func alwaysImportAccount(accountNumber int, collection string) bool { // If there is not a mapping for this collection // return false (don't import) for this account. if _, ok := AlwaysImportAccounts[collection]; !ok { return false } // Range through the list of accounts for the collection and // check if the passed in account number is equal to an // account number in the AlwaysImportAccounts mapping. // Return true (import) if there is a match. for _, account := range AlwaysImportAccounts[collection] { if account == accountNumber { return true } } return false } func alwaysImportMakeModel(deviceMakeModel string) bool { for _, model := range collection.AlwaysImportModels { if model == deviceMakeModel { return true } } return false } func hasSKU(deviceSKUs SKUData, targetSKUs []int) bool { for id := range deviceSKUs { for _, s2 := range targetSKUs { if id == strconv.Itoa(s2) { return true } } } return false } func hasSKUMap(deviceSKUs SKUData, targetSKUs map[int]string) (string, bool) { for id := range deviceSKUs { for ts, tv := range targetSKUs { if id == strconv.Itoa(ts) { return tv, true } } } return "", false } func setFlaggedIntegrationSKU(d *Device, deviceSKUs SKUData) bool { found := false for _, s2 := range collection.SKUsFlaggedIntegration { if val, ok := deviceSKUs[strconv.Itoa(s2)]; ok { for _, attr := range val.Attr { if strings.Contains(strings.ToLower(attr.Name), "device") { if val, err := strconv.Atoi(attr.AttrValue); err == nil { found = true d.Linked.ID = val } } if strings.Contains(strings.ToLower(attr.Name), "account") { if val, err := strconv.Atoi(attr.AttrValue); err == nil { found = true d.Linked.AccountNumber = val } } } } } return found } func setDeviceDetailsSKU(d *Device, deviceSKUs SKUData) bool { for _, s2 := range collection.SKUsDeviceDetails { if val, ok := deviceSKUs[strconv.Itoa(s2)]; ok { for _, attr := range val.Attr { if strings.Contains(strings.ToLower(attr.Name), "device details") { split := strings.SplitN(attr.AttrValue, "-", 2) if len(split) == 2 { if accountID, err := strconv.Atoi(split[0]); err == nil { d.Linked.AccountNumber = d.Account.Number d.Linked.AccountName = d.Account.Name d.Account.Number = accountID d.Account.Name = split[1] return true } } } } } } return false } // Environments function will gather a list of unique environments // based on a device list. func Environments(dl []Device) (env map[string]bool) { env = make(map[string]bool) for _, d := range dl { if d.Environment.Number == "" { continue } if _, ok := env[d.Environment.Number]; !ok { env[d.Environment.Number] = true } } return } // FilterCabinets allows for a custom scheme to be passed into the // client configuration. func FilterCabinets(devices []Device, filters []mongo.EBIFilters, col string, physicalOnly bool) []Device { // If there are no filters, we can return the // original device list. if len(filters) == 0 { return devices } logrus.Infof("found %d cabinet filters. applying filter logic", len(filters)) var ( // Create a map of cabs so that we can do a key compare later. cabs = make(map[string]bool) // Final device list that we will return after all the janky-ass logic. finalDeviceList = make([]Device, 0) // Environment list to build envList = make(map[string]bool) // Account list to build accountList = make(map[int]bool) ) // Generate the cabinet map for _, cab := range filters { if _, ok := cabs[cab.Value]; !ok { cabs[cab.Value] = true } } // Range through the devices returned from DMart and // and add the devices in selected cabinets to the final // device list. for _, device := range devices { if device.Environment.Current.Container == "" { continue } if _, ok := cabs[device.Environment.Current.Container]; ok { finalDeviceList = append(finalDeviceList, device) } } // If the physicalOnly flag is set, return the final device list // here, before adding in devices associated with the environment. if physicalOnly { return finalDeviceList } // Now, we need to go back through the list of devices in cabinets, // and pull our unique environment numbers and account numbers. for _, device := range finalDeviceList { if _, ok := accountList[device.Account.Number]; !ok && device.Account.Number != 0 { accountList[device.Account.Number] = true } if device.Environment.Number == "" { continue } if _, ok := envList[device.Environment.Number]; !ok { envList[device.Environment.Number] = true } } // AND FINALLY, we go back through the original device list, and grab any devices // that are in any of the environment lists for _, device := range devices { // If it's not in a cabinet that's part of the filter list // (because we already added those) if _, ok := cabs[device.Environment.Current.Container]; !ok { // BUT, it is in the environment list, then we can add it // to the final device list. if _, ok := envList[device.Environment.Number]; ok { finalDeviceList = append(finalDeviceList, device) continue } // OR, if it's the correct account number and the device make_model is one of the // types we should auto-import regardless of environment, then add to the list. if _, ok := accountList[device.Account.Number]; ok { if alwaysImportMakeModel(device.Device.MakeModel) { finalDeviceList = append(finalDeviceList, device) continue } if alwaysImportAccount(device.Account.Number, col) { finalDeviceList = append(finalDeviceList, device) } } } } logrus.Infof("found %d total devices after applying filter logic", len(finalDeviceList)) return finalDeviceList }