Unanticipated configuration has been sent to CPEs

Hello People,

Appreciate your efforts in developing such a great solution.

Background

We are currently provisioning around 15,000 CPEs using GenieACS. To be honest the current implementation of the ACS has not been updated. The current version is 1.2.0 (I know this is bad).

There was an incident recently that the ACS was sending the wrong/unanticipated configuration to some of the CPEs.

We are provisioning our VoIP configuration using the ACS. A provisioning script calls an extension script which is supposed to GET the VoIP configs from a REST service.

Incident

We configured the provisioning script to be triggered periodically (2 PERIODIC ).

Then we observed some of the configurations have been sent to the CPEs which were not supposed to have them. In essence, if there is no VoIP configuration found, the provisioning script should be backed off.

function setVoipConf(lines){
    /* 
    If there is no VoIP config or if the extension script failed,
    Provisioning should be backed off
    */
    if (lines == null){
        log(`VOIP: No VoIP config found`);
        return;
    }
    ...

But in our case, almost 1,500 of CPEs, which were not supposed to have VoIP configurations been provisioned with VoIP configurations.

Interestingly, the logs are saying that the script has been returned before calling the declare() function yet sent the configurations, and more surprisingly, the Tag has not been set.

function setVoipConf(lines){
    /* 
    If there is no VoIP config or if the extension script failed,
    Provisioning should be backed off
    */
    if (lines == null){
        log(`VOIP: No VoIP config found`);
        return;
    }
    ...
    declare(`InternetGatewayDevice.Services.VoiceService.1.VoiceProfile.1.SIP.${attrPrifix}ProxyServer`, {value: timeStamp}, {value: i.ProxyServer});
    ...

    // Set VoIP TAG with the Carrier Code
    declare(`Tags.VoIP_${i.CarrierCode}`, null, { value:  true});
    }
}

So in this case, I am clueless about what went wrong though we could remove the configs manually using NBI.

Appreciate it if someone could share some insight on this behavior.

Thanks
~Krishan

The REST API object

# If configs are found
[
  [
    {
      "AuthPassword": "string",
      "AuthUserName": "string",
      "CarrierCode": "string",
      "CidNumber": "string",
      "OutboundProxy": "string",
      "OutboundProxyPort": 0,
      "ProxyServer": "string",
      "ProxyServerPort": 0,
      "RegistrarServer": "string",
      "RegistrarServerPort": 0,
      "ServiceID": 0
    }
  ]
]
# If configs are not found return 404

Extension

"use strict";

const http = require("https");
const winston = require('winston');
const now = new Date();
const logger = winston.createLogger({
    level: 'info',
    format: winston.format.json(),
    defaultMeta: { service: 'genieacs-ext', ext: "voip", timestamp: now.toISOString() },
    transports: [
      new winston.transports.File({ filename: '/var/log/genieacs/genieacs-ext-access.log' }),
    ],
  });


function getVoipConf(args, callback){
    //Read ARGS for SN and API
    let api = args[0];
    let sn = args[1];

    // Append "/" if it has not been passed
    if (api.charAt(api.length-1) != '/'){
        api = api.concat("/");
    }
    // HTTPS Request
    http.get(`${api}${sn}`, (res) => {
        if (res.statusCode !== 200){
            logger.warn(`Request failed (status code: ${res.statusCode})`, {serial: sn});
            return callback(null, null);
        }
        let rawData = "";
        res.on("data", (chunk) => (rawData += chunk));
        res.on("end", () => {
        let sip = JSON.parse(rawData);
        logger.info(`Request completed`, {serial: sn});
        callback(null, sip);
      });
    }).on("error", (err) => {
        logger.warn(`Uncaught exception`, {serial: sn});
        callback(null, null);
      });
}

exports.getVoipConf = getVoipConf;

Provisioning Script

// 0-Boot: VoIP Configuration
const devOui = declare("DeviceID.OUI", {value: 1}).value[0];
const devSn = declare("DeviceID.SerialNumber", {value: 1}).value[0];
const timeStamp = Date.now();

function setVoipConf(lines){
    /* 
    If there is no VoIP config or if the extension script failed,
    Provisioning should be backed off
    */
    if (lines == null){
        log(`VOIP: No VoIP config found`);
        return;
    }
    let attrPrifix = "";
    log(`VOIP: Setting SIP Configuration`);
    for(const i of lines){
        // Prefix to set Standby SIP Connection
        if(lines.indexOf(i) == 1){
            log(`VOIP: Setting Standby SIP Configuration`);
            attrPrifix = "X_CT-COM_Standby-";
        } 
   
declare(`InternetGatewayDevice.Services.VoiceService.1.VoiceProfile.1.SIP.${attrPrifix}ProxyServer`, {value: timeStamp}, {value: i.ProxyServer});
        declare(`InternetGatewayDevice.Services.VoiceService.1.VoiceProfile.1.SIP.${attrPrifix}RegistrarServer`, {value: timeStamp}, {value: i.RegistrarServer});
        declare(`InternetGatewayDevice.Services.VoiceService.1.VoiceProfile.1.SIP.${attrPrifix}ProxyServerPort`, {value: timeStamp}, {value: i.ProxyServerPort});
        declare(`InternetGatewayDevice.Services.VoiceService.1.VoiceProfile.1.SIP.${attrPrifix}RegistrarServerPort`, {value: timeStamp}, {value: i.RegistrarServerPort});

        // Phone Lines
        declare(`InternetGatewayDevice.Services.VoiceService.1.VoiceProfile.1.Line.${lines.indexOf(i)+1}.SIP.AuthUserName`, {value: timeStamp}, {value: i.AuthUserName});
        declare(`InternetGatewayDevice.Services.VoiceService.1.VoiceProfile.1.Line.${lines.indexOf(i)+1}.SIP.CidNumber`, {value: timeStamp}, {value: i.CidNumber});
        declare(`InternetGatewayDevice.Services.VoiceService.1.VoiceProfile.1.Line.${lines.indexOf(i)+1}.SIP.AuthPassword`, {value: timeStamp}, {value: i.AuthPassword});

        // Set VoIP TAG with the Carrier Code
        declare(`Tags.VoIP_${i.CarrierCode}`, null, { value:  true});
    }
}

// 1-BOOT: Set VoIP configs
log("VOIP: SET VOIP Configurations");
switch(devOui) { 
    // For Nokia
    case "089BB9": 
    	 log("VOIP: Nokia CPE");
        break;
    // For NetComm
    case  "18F145":
    	log("VOIP: NetComm CPE")
        setVoipConf(ext("voip_ext", "getVoipConf", "https://symbill-apis.example.com/api/v1/svc/config/voip",devSn));
        break;
    default: 
        //Foe Other CPEs
        log("VOIP: Other CPE");
        break;      
}