// 2020 Rackspace, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"): you may // not use this file except in compliance with the License. You may obtain // a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the // License for the specific language governing permissions and limitations // under the License. // Package discover contains the business logic to pull switchport data for a device and // interface with Rackspace systems to add to an environment. package discover import ( "fmt" "strconv" "github.rackspace.com/DCM/discover/common" "github.rackspace.com/DCM/discover/cmdb" "github.rackspace.com/DCM/discover/ecm" "github.rackspace.com/DCM/discover/log" "github.rackspace.com/DCM/discover/network" "github.rackspace.com/gorax/clients/dcxAutohost" "github.rackspace.com/gorax/passkey" ) type ( // Client struct is a aggregation client for the packages and interfaces // used in the discover methods. Client struct { cmdb cmdb.Getter ecm ecm.Updater net *network.Client key *passkey.Client } // Result type will be the details of the job that ran for the // discovery. Result struct { Discoveries []Discovery `json:"discovered_configs" bson:"discovered_configs"` SVIs []SVI `json:"infrastructure_segments" bson:"infrastructure_segments"` Errors []string `json:"errors" bson:"errors"` models []cmdb.Model `json:"-" bson:"-"` } // Discovery is a list of items discovered on a per-account basis. Discovery struct { Account int `json:"account_number" bson:"account_number"` Environments []Environment `json:"environments" bson:"environments"` Logs []string `json:"logs" bson:"logs"` cfg ecm.Config `json:"-" bson:"-"` } // Environment is a sub document of the Result type for discovered environments. Environment struct { Number string `json:"number" bson:"number"` Devices []Device `json:"devices" bson:"devices"` Translations []common.Translation `json:"vlan_translations" bson:"vlan_translations"` PrivateBlocks []string `json:"private_blocks" bson:"private_blocks"` PublicBlocks []string `json:"public_blocks" bson:"public_blocks"` } // SVI is a sub document of the Result type for discovered environments. SVI struct { Vlan int `json:"vlan_number" bson:"vlan_number"` Name string `json:"name" bson:"name"` SVIDevices []string `json:"source_svi_devices" bson:"source_svi_devices"` } // Device is a sub document of the Environment type for device details Device struct { ID int `json:"id" bson:"id"` LinkedID int `json:"cmdb_id" bson:"cmdb_id"` IsGateway bool `json:"is_gateway" bson:"is_gateway"` Model string `json:"make_model" bson:"make_model"` PortMaps dcxAutohost.DevicePortMap `json:"port_maps" bson:"port_maps"` } ) // NewDatapipeDCX is a helper initialization function that initializes the interfaces with // admin.mydatapipe as the CMDB and DCX tooling as the Environment Config Manager. func NewDatapipeDCX(key *passkey.Client) (c *Client, err error) { c = &Client{key: key} if c.cmdb, err = cmdb.New(key, cmdb.Datapipe); err != nil { return } if c.ecm, err = ecm.New(key, ecm.DCXTools); err != nil { return } if c.net, err = network.New(network.WithPasskey(key), network.WithRefresh(3600)); err != nil { return } return } // Run method will execute the processes for the client using the // interfaces defined in the client initialization. // // This process runs, at a high-level: // 1.) For each unique ID, retrieve data able the record // 2.) For each record, gather network configuration // 3.) For each record, update EM with the network information func (c *Client) Run(uniqueIDs, vlansExcluded []int, update bool) *Result { // Make a response to return to the caller with the results of this // discovery run. retVal := &Result{ models: make([]cmdb.Model, 0), Discoveries: make([]Discovery, 0), SVIs: make([]SVI, 0), Errors: make([]string, 0), } // Creating a buffered channel that will block when we have reached the // passed in limit for concurrency semChan := make(chan struct{}, 10) // Create a non-blocking channel that will receive all the responses // back from validate. resChan := make(chan cmdb.Model) // Close the channels when this function has exited. defer func() { close(semChan) close(resChan) }() // 1.) For each unique ID, retrieve data able the record for _, id := range uniqueIDs { go func(id int) { semChan <- struct{}{} m, err := c.cmdb.Get(strconv.Itoa(id)) if err != nil { retVal.Errors = append(retVal.Errors, err.Error()) return } // now we can send the result struct through the resultsChan resChan <- m // Now that we're done, pull an empty struct from the channel, // freeing up one spot for another routine to take. <-semChan }(id) } // Receive the results from the resChan as the come in. for i := 0; i < len(uniqueIDs); i++ { result := <-resChan retVal.models = append(retVal.models, result) } // 2.) For each record, gather network configuration for i := range retVal.models { if err := c.net.ConfigsForPortsParsed(retVal.models[i].Environment.Current.PortsParsed); len(err) != 0 { for _, er := range err { if er.Error() != "" { retVal.Errors = append(retVal.Errors, fmt.Sprintf("[%d -> %d] %s", retVal.models[i].ID, retVal.models[i].Linked.ID, er)) } } continue } for x := range retVal.models[i].Environment.Current.PortsParsed { for _, vlanExcluded := range vlansExcluded { if retVal.models[i].Environment.Current.PortsParsed[x].Config.AccessVlan == vlanExcluded { retVal.models[i].Environment.Current.PortsParsed[x].Config.AccessVlan = -1 } if retVal.models[i].Environment.Current.PortsParsed[x].Config.NativeVlan == vlanExcluded { retVal.models[i].Environment.Current.PortsParsed[x].Config.NativeVlan = -1 } } filtered := make([]int, 0) for _, portVlan := range retVal.models[i].Environment.Current.PortsParsed[x].Config.VlanAllowed { shouldExclude := false for _, vlanExcluded := range vlansExcluded { if portVlan == vlanExcluded { shouldExclude = true break } } if !shouldExclude { filtered = append(filtered, portVlan) } } retVal.models[i].Environment.Current.PortsParsed[x].Config.VlanAllowed = filtered } } // 3.) TODO: Discovery only operates off of a single zone. We need to break up devices by zone here // if we are going to discover in multiple zones. // 4.) For each group of account devices, update the ECM with the network information for acct, deviceGroup := range ByAccount(retVal.models) { discovery := Discovery{ Account: acct, Logs: make([]string, 0), Environments: make([]Environment, 0), } var err error if discovery.cfg, err = c.ecm.Build(deviceGroup, c.net.SwitchConfigs()); err != nil { retVal.Errors = append(retVal.Errors, fmt.Sprintf("[%d] %s", acct, err)) } retVal.Errors = append(retVal.Errors, discovery.cfg.Logs.Errors...) discovery.cfg.Print() upResult := c.ecm.Update(discovery.cfg, update) retVal.Append(discovery, upResult) } return retVal } // PortMapResult is the response time for just mapping ports type PortMapResult struct { PortMaps []dcxAutohost.DevicePortMap `json:"port_maps"` Errors []string `json:"errors"` } // RunPortMaps method will execute a subset of the discovery logic to only apply and // write port-mappings for the devices. func (c *Client) RunPortMaps(uniqueIDs []int, update bool) PortMapResult { // Make a response to return to the caller with the results of this // discovery run. colDevices := make([]cmdb.Model, 0) res := PortMapResult{ PortMaps: make([]dcxAutohost.DevicePortMap, 0), Errors: make([]string, 0), } db, err := common.NewDB(c.key) if err != nil { res.Errors = append(res.Errors, err.Error()) return res } // Creating a buffered channel that will block when we have reached the // passed in limit for concurrency semChan := make(chan struct{}, 10) // Create a non-blocking channel that will receive all the responses // back from validate. resChan := make(chan cmdb.Model) // Close the channels when this function has exited. defer func() { close(semChan) close(resChan) }() // 1.) For each unique ID, retrieve data able the record for _, id := range uniqueIDs { go func(id int) { semChan <- struct{}{} m, err := c.cmdb.Get(strconv.Itoa(id)) if err != nil { res.Errors = append(res.Errors, err.Error()) return } // now we can send the result struct through the resultsChan resChan <- m // Now that we're done, pull an empty struct from the channel, // freeing up one spot for another routine to take. <-semChan }(id) } // Receive the results from the resChan as the come in. for i := 0; i < len(uniqueIDs); i++ { result := <-resChan colDevices = append(colDevices, result) } // 2.) For each record, gather network configuration for _, record := range colDevices { // This will eliminate any virtual devices or other devices with no port assignments // from attempting to build a port-map. if len(record.Environment.Current.PortsParsed) == 0 { continue } res.PortMaps = append(res.PortMaps, record.PortMap) if !update { continue } if err := db.WritePortMaps(record.ID, record.PortMap); err != nil { res.Errors = append(res.Errors, fmt.Sprintf("WritePortMaps:%d:%s", record.ID, err)) } } return res } // ByAccount will group the list of devices passed in by similar account numbers. // Return value will be a map of {account_number: []Devices} func ByAccount(d []cmdb.Model) map[int][]cmdb.Model { r := make(map[int][]cmdb.Model) for i := range d { if _, ok := r[d[i].Account.Number]; !ok { r[d[i].Account.Number] = make([]cmdb.Model, 0) } r[d[i].Account.Number] = append(r[d[i].Account.Number], d[i]) } return r } // Mapper maps the response structure from the update method into something // more acceptable for a API response func (r *Result) Append(d Discovery, u log.Collector) { for _, l := range u.Logs { if l.Type <= log.InfoLvl { if common.StringInSlice(l.String(), d.Logs) { continue } d.Logs = append(d.Logs, l.String()) } } for _, u := range d.cfg.Environments { d.Environments = append(d.Environments, Environment{ Number: u.Number, Devices: DeviceMapper(u), Translations: u.Translations, PrivateBlocks: PrivateCIRDMapper(u), PublicBlocks: PublicCIRDMapper(u), }) } for _, u := range d.cfg.PublicVlans { r.SVIs = append(r.SVIs, SVI{ Vlan: u.Vlan, Name: u.SegmentName(), SVIDevices: u.SVIDevices, }) } for i := range u.Errors { if common.StringInSlice(u.Errors[i], r.Errors) { continue } r.Errors = append(r.Errors, u.Errors[i]) } r.Discoveries = append(r.Discoveries, d) } // DeviceMapper maps the devices within a network environment into a device list // that is more readable/acceptable to a standard API contract func DeviceMapper(e ecm.NetEnv) []Device { r := make([]Device, 0) for _, d := range e.Devices { if deviceExist(d, r) { continue } r = append(r, Device{ ID: d.ID, LinkedID: d.Linked.ID, IsGateway: d.IsGateway, Model: d.Device.Device.MakeModel, PortMaps: d.PortMap, }) } for _, d := range e.Gateways { if deviceExist(d, r) { continue } r = append(r, Device{ ID: d.ID, LinkedID: d.Linked.ID, IsGateway: d.IsGateway, Model: d.Device.Device.MakeModel, PortMaps: d.PortMap, }) } return r } func deviceExist(d cmdb.Model, ds []Device) bool { for _, i := range ds { if i.ID == d.ID { return true } } return false } // PrivateCIRDMapper maps the private ip block within a network environment into a slice of strings // that is more readable/acceptable to a standard API contract func PrivateCIRDMapper(e ecm.NetEnv) []string { r := make([]string, 0) for _, bk := range e.PrivateBlocks { r = append(r, bk.CIDR) } return r } // PublicCIRDMapper maps the public ip block within a network environment into a slice of strings // that is more readable/acceptable to a standard API contract func PublicCIRDMapper(e ecm.NetEnv) []string { r := make([]string, 0) for _, bk := range e.RoutedBlocks { for _, dest := range bk.Dest { r = append(r, dest.CIDR()) } } return r }