POAP and Ansible integration part 4

In the last part of the series I will look at the boot process of a POAP installation
First thing to do is run the playbook to populate the tftpboot folder and create all the files.

root@debian-8:/home/poap/poap# ansible-playbook site.yml

PLAY [Generate access switch files] *******************************************

GATHERING FACTS ***************************************************************
ok: [localhost]

TASK: [dhcpd | Generate dhcpd main config files] ******************************
ok: [localhost]

TASK: [dhcpd | include_vars globals/poap_clients.yml] *************************
ok: [localhost]

TASK: [dhcpd | create client dhcpd config files] ******************************
ok: [localhost]

TASK: [tftpd | Generate poap files] *******************************************
changed: [localhost] => (item={'name': 'switch1', 'ip': '192.168.4.201', 'mask': '255.255.255.0', 'mac': '0:0:0:50:60:50', 'bootfile': 'switch1.py', 'tftp_server': '192.168.3.254', 'serial': 'JAC0001', 'type': 'n3k', 'gateway': '192.168.4.254', 'software': '6.0.2.U3.5'})
changed: [localhost] => (item={'name': 'switch2', 'ip': '192.168.4.202', 'mask': '255.255.255.0', 'mac': '0:0:0:50:60:51', 'bootfile': 'switch2.py', 'tftp_server': '192.168.3.254', 'serial': 'JAC0002', 'type': 'n3k', 'gateway': '192.168.4.254', 'software': '6.0.2.U3.5'})

TASK: [tftpd | Generate tftpd files] ******************************************
changed: [localhost] => (item={'name': 'switch1', 'ip': '192.168.4.201', 'mask': '255.255.255.0', 'mac': '0:0:0:50:60:50', 'bootfile': 'switch1.py', 'tftp_server': '192.168.3.254', 'serial': 'JAC0001', 'type': 'n3k', 'gateway': '192.168.4.254', 'software': '6.0.2.U3.5'})
changed: [localhost] => (item={'name': 'switch2', 'ip': '192.168.4.202', 'mask': '255.255.255.0', 'mac': '0:0:0:50:60:51', 'bootfile': 'switch2.py', 'tftp_server': '192.168.3.254', 'serial': 'JAC0002', 'type': 'n3k', 'gateway': '192.168.4.254', 'software': '6.0.2.U3.5'})

TASK: [tftpd | Copy NXOS Files] ***********************************************
changed: [localhost] => (item={'name': 'n3000-uk9-system.6.0.2.U3.5.bin'})
changed: [localhost] => (item={'name': 'n3000-uk9-kickstart.6.0.2.U3.5.bin'})

NOTIFIED: [tftpd | generate md5] **********************************************
changed: [localhost] => (item={u'src': u'/root/.ansible/tmp/ansible-tmp-1438835621.38-10613105394176/source', u'md5sum': u'719eba727a731782d9bf91501f8f0754', u'group': u'root', u'uid': 0, u'dest': u'/srv/tftp/conf_JAC0001.cfg', u'changed': True, 'item': {'name': 'switch1', 'ip': '192.168.4.201', 'mask': '255.255.255.0', 'mac': '0:0:0:50:60:50', 'bootfile': 'switch1.py', 'tftp_server': '192.168.3.254', 'serial': 'JAC0001', 'type': 'n3k', 'gateway': '192.168.4.254', 'software': '6.0.2.U3.5'}, u'state': u'file', u'gid': 0, u'mode': u'0644', 'invocation': {'module_name': u'template', 'module_args': u'src=n3k.j2 dest=/srv/tftp/conf_JAC0001.cfg'}, u'owner': u'root', u'size': 329})
changed: [localhost] => (item={u'src': u'/root/.ansible/tmp/ansible-tmp-1438835621.43-76402414020639/source', u'md5sum': u'a2e2a37799030c77364fa41755ef270a', u'group': u'root', u'uid': 0, u'dest': u'/srv/tftp/conf_JAC0002.cfg', u'changed': True, 'item': {'name': 'switch2', 'ip': '192.168.4.202', 'mask': '255.255.255.0', 'mac': '0:0:0:50:60:51', 'bootfile': 'switch2.py', 'tftp_server': '192.168.3.254', 'serial': 'JAC0002', 'type': 'n3k', 'gateway': '192.168.4.254', 'software': '6.0.2.U3.5'}, u'state': u'file', u'gid': 0, u'mode': u'0644', 'invocation': {'module_name': u'template', 'module_args': u'src=n3k.j2 dest=/srv/tftp/conf_JAC0002.cfg'}, u'owner': u'root', u'size': 329})

NOTIFIED: [tftpd | generate md5 nxos] *****************************************
changed: [localhost] => (item={u'src': u'/root/.ansible/tmp/ansible-tmp-1438835621.5-27092330543238/source', u'md5sum': u'f4e9e6a8d205febcaae5a413a0edd82b', u'group': u'root', u'uid': 0, u'dest': u'/srv/tftp/n3000-uk9-system.6.0.2.U3.5.bin', u'changed': True, 'item': {'name': 'n3000-uk9-system.6.0.2.U3.5.bin'}, u'state': u'file', u'gid': 0, u'mode': u'0666', 'invocation': {'module_name': u'copy', 'module_args': u'src=n3000-uk9-system.6.0.2.U3.5.bin dest=/srv/tftp/n3000-uk9-system.6.0.2.U3.5.bin mode=0666'}, u'owner': u'root', u'size': 32})
changed: [localhost] => (item={u'src': u'/root/.ansible/tmp/ansible-tmp-1438835621.57-161210011896998/source', u'md5sum': u'7d9265513b44bf23a07912f21167af58', u'group': u'root', u'uid': 0, u'dest': u'/srv/tftp/n3000-uk9-kickstart.6.0.2.U3.5.bin', u'changed': True, 'item': {'name': 'n3000-uk9-kickstart.6.0.2.U3.5.bin'}, u'state': u'file', u'gid': 0, u'mode': u'0666', 'invocation': {'module_name': u'copy', 'module_args': u'src=n3000-uk9-kickstart.6.0.2.U3.5.bin dest=/srv/tftp/n3000-uk9-kickstart.6.0.2.U3.5.bin mode=0666'}, u'owner': u'root', u'size': 35})

NOTIFIED: [tftpd | add md5] ***************************************************
changed: [localhost] => (item={u'src': u'/root/.ansible/tmp/ansible-tmp-1438835621.2-187701580008017/source', u'md5sum': u'22256ae23becfa917ac20c03c0f3c17b', u'group': u'root', u'uid': 0, u'dest': u'/srv/tftp/switch1.py.md5', u'changed': True, 'item': {'name': 'switch1', 'ip': '192.168.4.201', 'mask': '255.255.255.0', 'mac': '0:0:0:50:60:50', 'bootfile': 'switch1.py', 'tftp_server': '192.168.3.254', 'serial': 'JAC0001', 'type': 'n3k', 'gateway': '192.168.4.254', 'software': '6.0.2.U3.5'}, u'state': u'file', u'gid': 0, u'mode': u'0644', 'invocation': {'module_name': u'template', 'module_args': u'src=poap_n3k.j2 dest=/srv/tftp/switch1.py.md5'}, u'owner': u'root', u'size': 38617})
changed: [localhost] => (item={u'src': u'/root/.ansible/tmp/ansible-tmp-1438835621.29-188072396475654/source', u'md5sum': u'22256ae23becfa917ac20c03c0f3c17b', u'group': u'root', u'uid': 0, u'dest': u'/srv/tftp/switch2.py.md5', u'changed': True, 'item': {'name': 'switch2', 'ip': '192.168.4.202', 'mask': '255.255.255.0', 'mac': '0:0:0:50:60:51', 'bootfile': 'switch2.py', 'tftp_server': '192.168.3.254', 'serial': 'JAC0002', 'type': 'n3k', 'gateway': '192.168.4.254', 'software': '6.0.2.U3.5'}, u'state': u'file', u'gid': 0, u'mode': u'0644', 'invocation': {'module_name': u'template', 'module_args': u'src=poap_n3k.j2 dest=/srv/tftp/switch2.py.md5'}, u'owner': u'root', u'size': 38617})

PLAY RECAP ********************************************************************
localhost                  : ok=10   changed=6    unreachable=0    failed=0

root@debian-8:/home/poap/poap#

There were no changes required for the DHCP server but as I removed all files from the tftp root all files were created or copied in case of the NXOS files.

root@debian-8:/home/poap/poap# ls -l /srv/tftp/
root@debian-8:/home/poap/poap# ls -l /srv/tftp/
total 192
-rw-r--r-- 1 root root   329 Aug  6 06:24 conf_JAC0001.cfg
-rw-r--r-- 1 root root    40 Aug  6 06:24 conf_JAC0001.cfg.md5
-rw-r--r-- 1 root root   329 Aug  6 06:24 conf_JAC0002.cfg
-rw-r--r-- 1 root root    40 Aug  6 06:24 conf_JAC0002.cfg.md5
-rw-rw-rw- 1 root root    35 Aug  6 06:24 n3000-uk9-kickstart.6.0.2.U3.5.bin
-rw-r--r-- 1 root root    40 Aug  6 06:24 n3000-uk9-kickstart.6.0.2.U3.5.bin.md5
-rw-rw-rw- 1 root root    32 Aug  6 06:24 n3000-uk9-system.6.0.2.U3.5.bin
-rw-r--r-- 1 root root    40 Aug  6 06:24 n3000-uk9-system.6.0.2.U3.5.bin.md5
-rw-r--r-- 1 root root 38653 Aug  6 06:24 switch1.py
-rw-r--r-- 1 root root 38612 Aug  6 06:24 switch1.py.md5
-rw-r--r-- 1 root root 38651 Aug  6 06:24 switch2.py
-rw-r--r-- 1 root root 38610 Aug  6 06:24 switch2.py.md5
root@debian-8:/home/poap/poap#

At my dev system at home. I didn’t have the NXOS files available So i just created bogus files for demonstration purposes. The boot process below did use the correct software images.
Now all files are in place and the DHCP server is ready it is time to start the POAP process
To get a switch after it already has been configured back in poap mode a special boot option needs to be configured. Save the config and reboot the switch.

switch(config)# boot poap enable 

switch# copy running-config startup-config 

[########################################] 100%
Copy complete, now saving to disk (please wait)...


switch# reload

WARNING: This command will reboot the system

Do you want to continue? (y/n) [n] y

The system boots with the software 6.0.2.U2.2 (line 13) and POAP is enabled (line 26)

11:12:40 switch %PFMA-2-PFM_SYSTEM_RESET: Manual system restart from Command Line Interface

[ 1182.288358]  writing reset reason 9, 


(c) Copyright 2011, Cisco Systems.

N3000 BIOS v.1.2.0, Thu 08/25/2011, 03:37 PM 

989D9CB4B4B4999299A0A2A3A0A2A3B2                                                                           B2Version 2.00.1201. Copyright (C) 2009 American Megatrends, Inc.                 Press <DEL> or <F2> to enter setup.                                             
Loader Version pr-1.07
GRUB Loading stage2
Booting kickstart image: bootflash:/n3000-uk9-kickstart.6.0.2.U2.2.bin....
...............................................................................
............................Image verification OK

[    0.000000] Fastboot Memory at 0c100000 of size 201326592
ÿUsage: init 0123POST INIT Starts at Mon Aug 10 11:13:19 UTC 2015
Starting Nexus 3000 Platform POST.....
  Executing Mod 1 1 SEEPROM Test:...done (0 seconds)
  Executing Mod 1 1 GigE Port Test:.done (8 seconds)
  Executing Mod 1 1 PCIE Test:.................done (0 seconds)
  Mod 1 1 Post Completed Successfully
POST is completed
.r.r.r. done.
Bootstrapping via POAP overriding existing startup-config
Loading System Software Mon Aug 10 11:13:43 UTC 2015

System Software(/bootflash/n3000-uk9.6.0.2.U2.2.bin) Loaded Mon Aug 10 11:14:00 UTC 2015
ethernet switching mode

INIT: Entering runlevel: 3

Mounting other filesystems:  [  
Set name-type for VLAN subsystem. Should be visible in /proc/net/vlan/config
Added VLAN with VID == 4042 to IF -:muxif:-
11:14:07 switch %USER-0-SYSTEM_MSG: FAST REBOOT DISABLED - bcm_usd
11:14:07 switch %USER-2-SYSTEM_MSG: CLIS: loading cmd files begin  - clis
11:14:19 switch %USER-2-SYSTEM_MSG: CLIS: loading cmd files end  - clis
11:14:19 switch %USER-2-SYSTEM_MSG: CLIS: init begin  - clis
11:14:49 switch %USER-0-SYSTEM_MSG: Starting bcm_attach - bcm_usd
11:14:53 switch %USER-0-SYSTEM_MSG: Finished bcm_attach... - bcm_usd
11:15:05 switch %VDC_MGR-2-VDC_ONLINE: vdc 1 has come online 
Starting Power On Auto Provisioning...Done

Obviously we do not want to abort POAP. So we wait until the device does a DHCP request on its management port which happens on line 9 and after about 25 seconds the switch decided on use this offer and continue the process (line 10).

The bootfile is downloaded and execution starts. Not sur why on line 23 it is stated that the MD5SUM is not verified because and incorrect MD5 in the file results in a failed boot process. All other messages are self explanatory.

Abort Power On Auto Provisioning and continue with normal setup ?(yes/no)[n]: 

11:15:09 switch %POAP-2-POAP_INITED: S/N[JAC0001]-MAC[...] - POAP process initialized
11:30:18 switch %POAP-2-POAP_INFO:   - Abort Power On Auto Provisioning and continue with normal setup ?(yes/no)[n]: 
11:30:52 switch %POAP-2-POAP_INFO: S/N[JAC0001]-MAC[...] - USB disk not detected
11:30:52 switch %POAP-2-POAP_DHCP_DISCOVER_START: S/N[JAC0001]-MAC[...] - POAP DHCP Discover phase started
11:30:52 switch %POAP-2-POAP_INFO:   - Abort Power On Auto Provisioning and continue with normal setup ?(yes/no)[n]: 
11:45:16 switch %POAP-2-POAP_INFO:   - Abort Power On Auto Provisioning and continue with normal setup ?(yes/no)[n]: 
11:45:17 switch %POAP-2-POAP_INFO: S/N[JAC0001]-MAC[...] - Valid DHCP OFFER received from 192.168.3.254
11:45:43 switch %POAP-2-POAP_INFO: S/N[JAC0001]-MAC[...] - Using DHCP, information received over mgmt0 from 192.168.3.254
11:45:43 switch %POAP-2-POAP_INFO: S/N[JAC0001]-MAC[...] - Assigned Host Name: switch1
11:45:43 switch %POAP-2-POAP_INFO: S/N[JAC0001]-MAC[...] - Assigned IP address: 192.168.4.1
11:45:43 switch %POAP-2-POAP_INFO: S/N[JAC0001]-MAC[...] - Netmask: 255.255.255.0
11:45:43 switch %POAP-2-POAP_INFO: S/N[JAC0001]-MAC[...] - DNS Server: 192.168.3.254
11:45:43 switch %POAP-2-POAP_INFO: S/N[JAC0001]-MAC[...] - Default Gateway: 192.168.4.254
11:45:43 switch %POAP-2-POAP_INFO: S/N[JAC0001]-MAC[...] - Script Server: 192.168.3.254
11:45:43 switch %POAP-2-POAP_INFO: S/N[JAC0001]-MAC[...] - Script Name: switch1.py
11:45:54 switch %POAP-2-POAP_INFO: S/N[JAC0001]-MAC[...] - The POAP Script download has started
11:45:54 switch %POAP-2-POAP_INFO: S/N[JAC0001]-MAC[...] - The POAP Script is being downloaded from [copy tftp://192.168.3.254/switch1.py bootflash:script.sh vrf management ]
11:45:55 switch %POAP-2-POAP_SCRIPT_DOWNLOADED: S/N[JAC0001]-MAC[...] - Successfully downloaded POAP script file
11:45:55 switch %POAP-2-POAP_INFO: S/N[JAC0001]-MAC[...] - Default script timeout value:900 in script file
11:45:55 switch %POAP-2-POAP_INFO: S/N[JAC0001]-MAC[...] - Script file size 38968, MD5 checksum f512efd4bf9e962d22aed20305cb60cc
11:45:55 switch %POAP-2-POAP_SCRIPT_STARTED_MD5_NOT_VALIDATED: S/N[JAC0001]-MAC[...] - POAP script execution started(MD5 not validated)
11:45:55 switch %POAP-2-POAP_INFO: S/N[JAC0001]-MAC[...] - script timeout value:900 sec
11:45:56 switch %USER-1-SYSTEM_MSG: S/N[JAC0001]-MAC[...] - INFO: Selected config filename (serial-nb) : conf_JAC0001.cfg - script.sh
11:45:56 switch %USER-1-SYSTEM_MSG: S/N[JAC0001]-MAC[...] - CLI : dir bootflash: - script.sh
11:45:56 switch %USER-1-SYSTEM_MSG: S/N[JAC0001]-MAC[...] - INFO: free space is 1026448 kB - script.sh
11:45:56 switch %USER-1-SYSTEM_MSG: S/N[JAC0001]-MAC[...] - INFO:#Starting Copy of Config File - script.sh
11:46:00 switch %USER-1-SYSTEM_MSG: S/N[JAC0001]-MAC[...] - INFO: Completed Copy of Config File - script.sh
11:46:00 switch %USER-1-SYSTEM_MSG: S/N[JAC0001]-MAC[...] - INFO:#Check md5 of Configuration File - script.sh
11:46:04 switch %AUTHPRIV-1-SYSTEM_MSG:     root : can't get hostname - sudo
11:46:04 switch %USER-1-SYSTEM_MSG: S/N[JAC0001]-MAC[...] - CLI : show file volatile:conf_JAC0001.cfg.md5.poap_md5 - script.sh
11:46:04 switch %USER-1-SYSTEM_MSG: S/N[JAC0001]-MAC[...] - INFO: md5sum c006ba91100eff0407c4b7bba7b7ec88 (.md5 file) - script.sh
11:46:04 switch %USER-1-SYSTEM_MSG: S/N[JAC0001]-MAC[...] - CLI : show file poap_replay.cfg md5sum - script.sh
11:46:04 switch %USER-1-SYSTEM_MSG: S/N[JAC0001]-MAC[...] - INFO: md5sum c006ba91100eff0407c4b7bba7b7ec88 (recalculated) - script.sh
11:46:04 switch %USER-1-SYSTEM_MSG: S/N[JAC0001]-MAC[...] - INFO: Split config invoked.... - script.sh
11:46:04 switch %USER-1-SYSTEM_MSG: S/N[JAC0001]-MAC[...] - CLI : delete poap_replay.cfg - script.sh
11:46:04 switch %USER-1-SYSTEM_MSG: S/N[JAC0001]-MAC[...] - CLI : delete poap_1.cfg - script.sh
11:46:05 switch %USER-1-SYSTEM_MSG: S/N[JAC0001]-MAC[...] - INFO:#Starting Copy of Kickstart Image - script.sh
11:46:12 switch %USER-1-SYSTEM_MSG: S/N[JAC0001]-MAC[...] - INFO: Completed Copy of Kickstart Image - script.sh
11:46:12 switch %USER-1-SYSTEM_MSG: S/N[JAC0001]-MAC[...] - INFO:#Check md5 of kickstart image - script.sh
11:46:17 switch %USER-1-SYSTEM_MSG: S/N[JAC0001]-MAC[...] - CLI : show file volatile:n3000-uk9-kickstart.6.0.2.U3.5.bin.md5.poap_md5 - script.sh
11:46:17 switch %USER-1-SYSTEM_MSG: S/N[JAC0001]-MAC[...] - INFO: md5sum fea16185a6104abba5179a73e438ef29 (.md5 file) - script.sh
11:46:17 switch %USER-1-SYSTEM_MSG: S/N[JAC0001]-MAC[...] - CLI : show file bootflash:/kickstart.img.new md5sum - script.sh
11:46:19 switch %USER-1-SYSTEM_MSG: S/N[JAC0001]-MAC[...] - INFO: md5sum fea16185a6104abba5179a73e438ef29 (recalculated) - script.sh
11:46:19 switch %USER-1-SYSTEM_MSG: S/N[JAC0001]-MAC[...] - CLI : move bootflash:/kickstart.img.new bootflash:/n3000-uk9-kickstart.6.0.2.U3.5.bin - script.sh
11:46:19 switch %USER-1-SYSTEM_MSG: S/N[JAC0001]-MAC[...] - INFO:#Starting Copy of System Image - script.sh
11:46:44 switch %USER-1-SYSTEM_MSG: S/N[JAC0001]-MAC[...] - INFO: Completed Copy of System Image - script.sh
11:46:45 switch %USER-1-SYSTEM_MSG: S/N[JAC0001]-MAC[...] - INFO:#Check md5 of system image - script.sh
11:46:51 switch %AUTHPRIV-1-SYSTEM_MSG:     root : can't get hostname - sudo
11:46:51 switch %USER-1-SYSTEM_MSG: S/N[JAC0001]-MAC[...] - CLI : show file volatile:n3000-uk9.6.0.2.U3.5.bin.md5.poap_md5 - script.sh
11:46:51 switch %USER-1-SYSTEM_MSG: S/N[JAC0001]-MAC[...] - INFO: md5sum 68e7c10e308864a5c8656c2c459b6897 (.md5 file) - script.sh
11:46:51 switch %USER-1-SYSTEM_MSG: S/N[JAC0001]-MAC[...] - CLI : show file bootflash:/system.img.new md5sum - script.sh
11:46:58 switch %USER-1-SYSTEM_MSG: S/N[JAC0001]-MAC[...] - INFO: md5sum 68e7c10e308864a5c8656c2c459b6897 (recalculated) - script.sh
11:46:58 switch %USER-1-SYSTEM_MSG: S/N[JAC0001]-MAC[...] - CLI : move bootflash:/system.img.new bootflash:/n3000-uk9.6.0.2.U3.5.bin - script.sh
11:46:59 switch %USER-1-SYSTEM_MSG: S/N[JAC0001]-MAC[...] - INFO: Entered get_cable_mgmt_file - script.sh
11:46:59 switch %USER-1-SYSTEM_MSG: S/N[JAC0001]-MAC[...] -  found cable file: 0  - script.sh
11:46:59 switch %USER-1-SYSTEM_MSG: S/N[JAC0001]-MAC[...] - INFO: No cable file specified - script.sh
11:46:59 switch %USER-1-SYSTEM_MSG: S/N[JAC0001]-MAC[...] - CLI : show version image bootflash:/n3000-uk9-kickstart.6.0.2.U3.5.bin - script.sh
11:47:09 switch %USER-1-SYSTEM_MSG: S/N[JAC0001]-MAC[...] - CLI : show version image bootflash:/n3000-uk9.6.0.2.U3.5.bin - script.sh
11:48:09 switch %USER-1-SYSTEM_MSG: S/N[JAC0001]-MAC[...] - INFO: Setting the boot variables - script.sh
11:48:09 switch %USER-1-SYSTEM_MSG: S/N[JAC0001]-MAC[...] - CLI : config terminal ; boot kickstart bootflash:/n3000-uk9-kickstart.6.0.2.U3.5.bin - script.sh
11:48:11 switch %USER-1-SYSTEM_MSG: S/N[JAC0001]-MAC[...] - CLI : config terminal ; boot system bootflash:/n3000-uk9.6.0.2.U3.5.bin - script.sh
11:48:14 switch %USER-1-SYSTEM_MSG: S/N[JAC0001]-MAC[...] - CLI : copy running-config startup-config - script.sh
11:48:19 switch %USER-1-SYSTEM_MSG: S/N[JAC0001]-MAC[...] - INFO: successful - script.sh
11:48:19 switch %USER-1-SYSTEM_MSG: S/N[JAC0001]-MAC[...] - CLI : copy bootflash:poap_2.cfg scheduled-config - script.sh
11:48:20 switch %USER-1-SYSTEM_MSG: S/N[JAC0001]-MAC[...] - ######### Copying the second scheduled cfg done ########## - script.sh
11:48:20 switch %USER-1-SYSTEM_MSG: S/N[JAC0001]-MAC[...] - INFO: Configuration successful - script.sh
11:48:20 switch %USER-1-SYSTEM_MSG: S/N[JAC0001]-MAC[...] - FINISH: Clean up files. - script.sh
11:48:20 switch %USER-1-SYSTEM_MSG: S/N[JAC0001]-MAC[...] - CLI : delete poap_2.cfg - script.sh
11:48:21 switch %POAP-2-POAP_SCRIPT_EXEC_SUCCESS: S/N[JAC0001]-MAC[...] - POAP script execution success
11:48:23 switch %PFMA-2-PFM_SYSTEM_RESET: Manual system restart from Command Line Interface

[ 2111.564321]  writing reset reason 9, 

The switch reboots after the succesfull POAP process and reboots with the specified software version and we are able to login with the username specified in the configuration file.

switch(config)# boot poap enable 

(c) Copyright 2011, Cisco Systems.

N3000 BIOS v.1.2.0, Thu 08/25/2011, 03:37 PM 

989D9CB4B4B4999299A0A2A3A0A2A3B2                                                                B2Version 2.00.1201. Copyright (C) 2009 American Megatrends, Inc.                 Press <DEL> or <F2> to enter setup.                                             
Loader Version pr-1.07
GRUB Loading stage2                                                                        Booting kickstart image: bootflash:/n3000-uk9-kickstart.6.0.2.U3.5.bin....
...............................................................................
...........................Image verification OK

[    0.000000] Fastboot Memory at 0c100000 of size 201326592
ÿUsage: init 0123POST INIT Starts at Mon Aug 10 11:49:02 UTC 2015
Starting Nexus 3000 Platform POST.....
  Executing Mod 1 1 SEEPROM Test:...done (0 seconds)
  Executing Mod 1 1 GigE Port Test:.done (8 seconds)
  Executing Mod 1 1 PCIE Test:.................done (0 seconds)
  Mod 1 1 Post Completed Successfully
POST is completed
.r.r.r. done.
Bootstrapping via POAP overriding existing startup-config
Loading System Software Mon Aug 10 11:49:22 UTC 2015

System Software(/bootflash/n3000-uk9.6.0.2.U3.5.bin) Loaded Mon Aug 10 11:49:43 UTC 2015
ethernet switching mode
cp: cannot stat `/isan/etc/capability.cap': No such file or directory

INIT: Entering runlevel: 3

Mounting other filesystems:  [  OK  ]


Set name-type for VLAN subsystem. Should be visible in /proc/net/vlan/config
Added VLAN with VID == 4042 to IF -:muxif:-
11:49:53  %USER-0-SYSTEM_MSG: FAST REBOOT DISABLED - bcm_usd
11:49:53  %USER-2-SYSTEM_MSG: CLIS: loading cmd files begin  - clis
11:50:04  %USER-2-SYSTEM_MSG: CLIS: loading cmd files end  - clis
11:50:04  %USER-2-SYSTEM_MSG: CLIS: init begin  - clis
11:50:27  %USER-0-SYSTEM_MSG: Starting bcm_attach - bcm_usd
11:50:31  %USER-0-SYSTEM_MSG: Finished bcm_attach... - bcm_usd
11:50:44  %VDC_MGR-2-VDC_ONLINE: vdc 1 has come online 
POAP - Waiting for Box Online...
POAP - Box is Online...
POAP - Applying scheduled configuration...
Copy complete, now saving to disk (please wait)...


[########################################] 100%
Copy complete, now saving to disk (please wait)...
Done

11:51:18 switch1 %SYSLOG-2-SYSTEM_MSG: POAP Completed - LOGIN

Nexus 3000 Switch
switch1 login: nettinkerer
Password: 
Cisco Nexus Operating System (NX-OS) Software
TAC support: http://www.cisco.com/tac
Copyright (c) 2002-2014, Cisco Systems, Inc. All rights reserved.
The copyrights to certain works contained in this software are
owned by other third parties and used and distributed under
license. Certain components of this software are licensed under
the GNU General Public License (GPL) version 2.0 or the GNU
Lesser General Public License (LGPL) Version 2.1. A copy of each
such license is available at
http://www.opensource.org/licenses/gpl-2.0.php and
http://www.opensource.org/licenses/lgpl-2.1.php

switch1#

 

As you can see POAP is very powerfull to quickly upgrade and configure a large number of new switches. It would also be possible to modify the playbook to use the configuration of a failed switch. Imagine sending a replacement switch to the datacenter, the field engineer repalces the switch. You only need to change one line in a YAML file, run the playbook and the POAP files are prepared and the DHCP server is reconfigured and restarted.

POAP and Ansible integration part 3

The third part of the series will be about all the files required for the boot process. The boot process follows the diagram below.

poap_process

All configuration files required for the boot process are generated by TFTPD role in the playbook. The tasks associated with this roles are defined in the YAML file.

---
- name: Generate poap files
  template: src=poap_n3k.j2 dest=/srv/tftp/{{item.name}}.py.md5
  register: poap_created
  with_items: clients
  notify: add md5
- name: Generate tftpd files
  template: src=n3k.j2 dest=/srv/tftp/cfg_{{item.serial}}.cfg
  register: config_created
  with_items: clients
  notify: generate md5
- name: Copy NXOS Files
  copy: src={{item.name}} dest=/srv/tftp/{{item.name}} mode=0666
  register: ios_copied
  with_items: nxosfiles
  notify: generate md5 nxos

The bootfile is a Python script based on an example which can be downloaded from CCO when you have correct entitlement. I have modified the Python script a bit and removed one bug which prevented the script to recognize the switch as a Nexus 3048. In the script all the details for the POAP process are specified.

  • software version
  • configuration file
  • download credentials
  • transfer method
  • download server

As I wanted to be flexible in the software version I used the templating system of Ansible to generate custom py files for booting.

The handler called when the py file changes is used to create the actual py file provided via the DHCP offer. In the actual file an extra line is added with the md5sum of the file without this extra line. When executed by the switch the python script will remove the line with the md5sum, calculate the md5sum and verify the script.  The handlers are specified in a separate YAML file

---
- name: generate md5
  shell: md5sum {{ item.dest }} | awk {'print "md5sum=" $1'} > {{item.dest}}.md5
  when: item.changed
  with_items: config_created.results

- name: generate md5 nxos
  shell: md5sum {{ item.dest }} | awk {'print "md5sum=" $1'} > {{item.dest}}.md5
  when: item.changed
  with_items: ios_copied.results

- name: add md5
  command: /bin/addmd5 {{ item.dest }}
  when: item.changed
  with_items: poap_created.results

The handler for the py file is add md5 This handler executes a bash script to calculate the md5, add it to the file and store it as a new py file without the md5 suffix. This is the file downloaded by the switch during the POAP process and needs to be supplied by the DHCP server als Boot file.

#!/bin/bash
#script to add md5sum to the second line of a Python file used for POAP files
md5=`md5sum $1 | awk {'print "#md5sum=" $1 '}`
poap_file=`echo $1 | sed "s/\.md5//"`
sed '2i'$md5 $1 > $poap_file

When the md5 of the bootfile matches with the md5 included in the file the complete script is executed. In the script the transfer method is specified. If another method than tftp is used for transfer of the configuration and software credentials need to be specified. Please be aware that these credentials will be sent unencrypted to the switch when the py file is transferred. The files to be transferred are also specified in the script. In this example the name of the configuration file to be downloaded is derived from the serial number. This can be seen in line 8 of the roles/tftpd/tasks/main.yml file.

The next task is to create the actual configuration files. This is pretty straightforward. More about this can be found In a previous blog on this site. The only special thing is the handler generate md5 which calculates the md5sum of the configuration file and places this value in a textfile. This textfile has the same name as the configuration file with an .md5 suffix. The format of the string is md5sum=12345abcdef. The Python script executed by the POAP process will download these file automatically and verify the MD5SUM.

The last task in the playbook copies all the NXOS images to the TFTP server. Again a handler is called to create the md5 files like with the configuration files.

It is important to realize that Ansible is indempodent. It will always strive to keep everything in a consistent state regardless of how many times a playbook is run.  This also means that files generated by Ansible must not be changed by hand. The next time the playbook is run the changes made by hand will be lost.

In the last blog in the series I will show how everything works together and the switch will do a POAP.

Part 4

POAP and Ansible integration part 2

In this part of the serie I will discuss the isc-dhcpd server configuration. isc-dhcpd is a DHCP server which is available on most linux distributions. It has many options but for this setup only a minimal configuration is required.

The directory layout  for the ansible-playbook for the DHCPD role

/home/poap/poap/
|-- globals
|   `-- poap_clients.yml
|-- roles
|   |-- dhcpd
|   |   |-- handlers
|   |   |   `-- main.yml
|   |   |-- tasks
|   |   |   `-- main.yml
|   |   |-- templates
|   |   |   |-- dhcpd.conf.j2
|   |   |   `-- static_clients.j2
|   |   `-- vars
|   |       `-- main.yml
|   `-- includes
`-- site.yml

The tasks for the DHCPD role are defined in roles/dhcpd/tasks/main.yml.

---
- name: Generate dhcpd main config files
  template: src=dhcpd.conf.j2 dest=/etc/dhcp/dhcpd.conf
  notify: restart dhcpd
- include_vars: globals/poap_clients.yml
- name: create client dhcpd config files
  template: src=static_clients.j2 dest=/etc/dhcp/static_clients
  notify: restart dhcpd

in role/dhcpd/vars/main.yml basic settings are configured for the DHCP server.

---
domain_name: home.local
domain_name_servers: 192.168.3.254
default_lease_time: 7200
max_lease_time: 14400
scopes:
 - subnet: 192.168.3.0
   netmask: 255.255.255.0
 - subnet: 192.168.4.0
   netmask: 255.255.255.0
   ranges:
   - range_start: 192.168.4.1
     range_end: 192.168.4.200
     routers: 192.168.4.254

I my lab I used two scopes and one range to allocate addresses from. These settings are used in the dhcpd.conf.j2 template to create the main dhcpd.conf

#jinja2: lstrip_blocks: True
ddns-update-style none;

# option definitions common to all supported networks...

option domain-name "{{domain_name}}";
option domain-name-servers {{domain_name_servers}};

default-lease-time {{default_lease_time}};
max-lease-time {{max_lease_time}};

{% for scope in scopes %}
subnet {{scope.subnet}} netmask {{scope.netmask}} {
  {% if scope.ranges is defined %}
  {% for range in scope.ranges %}
  range {{range.range_start}} {{range.range_end}};
  {% if range.routers is defined %}
  option routers {{range.routers}};
  {% endif %}
  {% endfor %}
  {% endif %}
}
{% endfor %}
include "/etc/dhcp/static_clients";

At the end of the configuration an additional configuration file called static_clients has been included, in which the reservations for the statich (POAP) clients are defined. I have placed these in a separate file for a reason. In a normal environment there would be at least two DHCP servers. Each server would be responsible for a part of the subnet to allocate address from. Or there would be a master/slave relation between the two servers which requires different configurations on both. The reservations however must be the same on both servers.

This template is used by the task Generate dhcpd main config files. The handler is instructs Ansible to restart the DHCPD service but only when the configuration has changed.
The next task is to include an additional YAML file globals_poap_clients.yml with data about the various poap clients. The file is placed in a different directory than the normal vars directory belonging to the role because it will also be used by the TFTPD role.

---
clients:
  - name: switch1
    mac: 0:0:0:50:60:50
    tftp_server: 192.168.3.254
    ip: 192.168.4.201
    mask: 255.255.255.0
    gateway: 192.168.4.254
    serial: JAC0001
    type: n3k
    software: 6.1.4
  - name: switch2
    mac: 0:0:0:50:60:51
    tftp_server: 192.168.3.254
    ip: 192.168.4.202
    mask: 255.255.255.0
    gateway: 192.168.4.254
    serial: JAC0002
    type: n3k
    software: 4.2

This files specifies two Nexus devices. The data is being used in the task create client dhcpd config files and fed to the template for the POAP clients.

#static clients
{% if clients is defined %}
{% for client in clients %}
host {{client.name}} {
  option host-name "{{client.name}}";
  option dhcp-client-identifier "\000{{client.serial}}";
  option bootfile-name "{{client.name}}.py";
  option tftp-server-name "{{client.tftp_server}}";
}
{% endfor %}
{% endif %}

This configuration will provide for each poap client:

  • Hostname
  • Bootfile
  • Bootserver

Settings like the IP address/mask/gateway/DNS are provided via the global scope. The ip details specified in the YAML file will be used for the generation of the actual switch configuration files.

Normally reservations are made based on the MAC address. In this setup I have chosen to make the reservation based on the serial of switch. This is possible because the serial is used as the client-identifier in the DHCP request. The serial of a new switch is often more easilly obtained than the mac address and I hate entering mac addresses as each vendor/tool requires a different format.

It took a Wireshark capture to get it working because Cisco prepends the client- identifier with an ASCII NULL. That is why the \000 in front of the {{client.serial}} is required on line 5

Again when dhcp settings have changed like adding a POAP client the DHCPD service will be restarted by Ansible.

After running the playbook the configuration for the DHCP server is generated.

ddns-update-style none;

# option definitions common to all supported networks...
option domain-name "home.local";
option domain-name-servers 192.168.3.254;

default-lease-time 7200;
max-lease-time 14400;

subnet 192.168.3.0 netmask 255.255.255.0 {
}
subnet 192.168.4.0 netmask 255.255.255.0 {
  range 192.168.4.1 192.168.4.200;
  option routers 192.168.4.254;
}
include "/etc/dhcp/static_clients";
#template for static clients
host switch1 {
  option dhcp-client-identifier "\000JAC0001";
  option host-name "switch1";
  option bootfile-name "JAC0001.py";
  option tftp-server-name "192.168.3.254";
}
host switch2 {
  option dhcp-client-identifier "\000JAC0002";
  option host-name "switch2";
  option bootfile-name "JAC0002.py";
  option tftp-server-name "192.168.3.254";
}

Overall the DHCP server configuration is pretty simple. In my lab the DHCP server is running on the same hosts as the ansible-scripts In a real world deployment this will most likely be different remote servers. How to configure Ansible to connect to remote DHCP servers is beyond the scope of this series but can be found on the internet easilly

This was part 2 of the series. In part 3 I will discuss about all the various files which need to be generated to make the POAP work.

Part 1

POAP and Ansible integration part 1

Everyone who has every installed a Nexus switch is familiar with the following message.

%POAP-2-POAP_INFO: Abort Power On Auto Provisioning and continue with normal setup ?(yes/no)[n]:

I always pressed y and be done with it. Since I have been using Ansible to create config files and to deploy Linux clients I have been wondering if I can do it all with Ansible. In a number of blogs I will describe  how to setup everything and never touch your console cable anymore. Please follow me on Twitter for the other blogs on this subject.

The flowchart for the setup is below

Poap_Ansible1

Everything is being specified in a number of YAML files. In the YAML files details about the POAP clients like, serial number, desired software version, hardware platform and ip details are specified. Also the basic DHCP server configuration parameters are specified in a YAML file.

The YAML files are used to create the following files via the templating system.

  • isc-dhcpd configuration files
  • bootfiles for the Nexus devices
  • configuration files for the Nexus devices

The creation of all these files has been split in two roles

DHCPD and TFTPD

In the next blog post I  will describe how the DHCPD role is responsible for the isc-dhcpd service.

 

Using Ansible to create config files

Recently I had to roll out a number of access switches. In the past I created the config files with either Excel/Word via a mailmerge or custom perl scripts. Both methods were not ideal. Mailmerge is inflexible and although I know my way around in Perl my colleagues often do not. After reading the excellent Ansible blog by Kirk Byers I gave it a try.

Ansible is primarily a tool like Chef and Puppet for server management. To make Ansible do something it has a concept named playbooks. A playbook defines which roles a specific host has. Each role has it specific tasks which need to be executed on that  hosts. For example a hosts has a role as DNS server . Tasks associated with this role could be make sure the latest version of Bind is installed and all the zone files are up to date. But also a task of creating the zone files by means of using a template system. This template system will be used to create the configuration files in this example

 Almost all files used by Ansible are written in the YAML format.
Below is the playbook used in this example.

---
- name: Generate access switch files
  hosts: localhost

  roles:
  - switch

Normally the tasks indicated by the roles would be executed on a remote host (remember the DNS server from above). For this example the files are generated on the same host as the Ansible script is being run this could also be a remote TFTP server for example.
The tasks belonging to the switch role of localhost are defined in a separate YAML file.

---
- name: Generate config files
  template: src=switch.j2 dest=/home/ansible/nettinkerer/config/{{item.hostname}}.
txt
  with_items: access_switches

The task executed on the local host is creating files based on the Jinja2 template. The variables being used are also defined in a YAML file. The template is being completed by looping over item of the dictionary access_switches

---
access_switches:
  - hostname: switch_1
    vpc_domain: 20
    core_node: core_1
    core_uplink: 1/0/10
    core_portchannel: 11
    vpc_roleprio: 4096
    vpc_peer_dest: 10.1.1.2
    vpc_peer_source: 10.1.1.1
    vpc_peernode: switch_2
    ip_vlan: 192.168.1.1
    snmp_location: Rack1
  - hostname: switch_2
    vpc_domain: 20
    core_node: core_1
    core_uplink: 2/0/10
    core_portchannel: 11
    vpc_roleprio: 8192
    vpc_peer_dest: 10.1.1.1
    vpc_peer_source: 10.1.1.2
    vpc_peernode: switch_2
    ip_vlan: 192.168.1.2
    snmp_location: Rack1

The contents of the Jinja2 file.

!template for access switch
feature lacp
feature udld
feature interface-vlan
feature vpc
no password strength-check
username nettinkerer password nettinkerer
username nettinkerer role network-admin
!
{% include "files/vlan" %}
!
{% include "/roles/includes/aaa" %}
!
{% include "files/stp" %}
!
hostname {{item.hostname}}
!
vpc domain {{item.vpc_domain}}
 peer-switch
 role priority {{item.vpc_roleprio}}
 system-priority 200
 peer-keepalive destination {{item.vpc_peer_dest}} source {{item.vpc_peer_source}}
 delay restore 300
 auto-recovery reload 900
!
interface ethernet1/49
 description {{item.vpc_peernode}}_E1/49
 channel-group 1 mode active
!
interface ethernet1/50
 description {{item.vpc_peernode}}_E1/50
 channel-group 1 mode active
!
interface Port-channel1
 description {{item.vpc_peernode}}_Po1
 switchport mode trunk
 spanning-tree port type network
 vpc peer-link
!
interface ethernet1/51
  description {{item.core_node}}_E{{item.core_uplink}}
  channel-group 2 mode active
!
interface Port-channel2
  description {{item.core_node}}_Po{{item.core_portchannel}}
  switchport mode trunk
  vpc 2
!
interface Vlan2
 description Management Interface
 ip address {{item.ip_vlan}} 255.255.255.0
 no shut
!
interface mgmt0
 ip address {{item.vpc_peer_source}} 255.255.255.252
!
vrf context management
 ip route 0.0.0.0 0.0.0.0 10.1.1.254
!
!
snmp-server location {{item.snmp_location}}
{% include "files/snmp" %}
!
end

This is a fairly simple Jinja2 file and is easy to read even without knowledge of the Jinja2 language. Everything between double curly brackets are variables which are being replaced with the actual value. Everything enclosed by a curly bracket and the percent sign is a function of the Jinja2 templating system. In this case a simple include for very static things like vlans and snmp stuff.
The directory layout for an Ansible script is very important. All files are expected to be found in specific directories. Below is the layout for this tutorial. Don’t worry about the router subdirectory for this moment.

.
|-- config
|-- roles
|   |-- includes
|   |   `-- aaa
|   |-- router
|   |   |-- tasks
|   |   |   `-- main.yml
|   |   |-- templates
|   |   |   |-- files
|   |   |   |   |-- snmp
|   |   |   |   |-- stp
|   |   |   |   `-- vlan
|   |   |   `-- router.j2
|   |   `-- vars
|   |       `-- main.yml
|   `-- switch
|       |-- tasks
|       |   `-- main.yml
|       |-- templates
|       |   |-- files
|       |   |   |-- snmp
|       |   |   |-- stp
|       |   |   `-- vlan
|       |   `-- switch.j2
|       `-- vars
|           `-- main.yml
`-- site.yml

The magic happens by running the playbook.

ansible@python-dev:~/nettinkerer$ ansible-playbook site.yml

PLAY [Generate access switch files] *******************************************

GATHERING FACTS ***************************************************************
ok: [localhost]

TASK: [switch | Generate config files] ****************************************
changed: [localhost] => (item={'core_uplink': '1/0/10', 'ip_vlan': '192.168.1.1', 'core_port                                                       channel': 11, 'hostname': 'switch_1', 'vpc_domain': 20, 'snmp_location': 'Rack1', 'vpc_peer_                                                       dest': '10.1.1.2', 'vpc_peer_source': '10.1.1.1', 'vpc_peernode': 'switch_2', 'core_node': '                                                       core_1', 'vpc_roleprio': 4096})
changed: [localhost] => (item={'core_uplink': '2/0/10', 'ip_vlan': '192.168.1.2', 'core_port                                                       channel': 11, 'hostname': 'switch_2', 'vpc_domain': 20, 'snmp_location': 'Rack1', 'vpc_peer_                                                       dest': '10.1.1.1', 'vpc_peer_source': '10.1.1.2', 'vpc_peernode': 'switch_2', 'core_node': '                                                       core_1', 'vpc_roleprio': 8192})

PLAY RECAP ********************************************************************
localhost                  : ok=2    changed=1    unreachable=0    failed=0

ansible@python-dev:~/nettinkerer$

and the configuration files can be found in the config directory.

ansible@python-dev:~/nettinkerer$ ls -l1 config/
total 8
-rw-r--r-- 1 ansible ansible 2352 Jul 28 21:31 switch_1.txt
-rw-r--r-- 1 ansible ansible 2352 Jul 28 21:31 switch_2.txt
ansible@python-dev:~/nettinkerer$

Although it might seem to be a lot of work to create all these YAML and Jinja2 files to generate a couple of configuration files it can save a lot of work later on. Imagine that you have generated 40 configurations and all of a sudden there is an additional vlan which needs to be included in all configurations. Now it is just a case of modifying one single file and generate all the configuration files by simply running the playbook again.