Configuring it

root@cloudinabox:/opt/cloudsimulation/zones# dladm create-bridge london
root@cloudinabox:/opt/cloudsimulation/zones# dladm create-bridge hamburg
root@cloudinabox:/opt/cloudsimulation/zones# dladm create-bridge singapore
root@cloudinabox:/opt/cloudsimulation/zones# dladm create-bridge newyork
root@cloudinabox:/opt/cloudsimulation/zones# dladm create-bridge sanfrancisco

root@cloudinabox:/opt/cloudsimulation/zones# dladm create-simnet londonsw1_255
root@cloudinabox:/opt/cloudsimulation/zones# dladm create-simnet sanfranciscosw1_255
root@cloudinabox:/opt/cloudsimulation/zones# dladm create-simnet newyorksw1_255
root@cloudinabox:/opt/cloudsimulation/zones# dladm create-simnet hamburgsw1_255
root@cloudinabox:/opt/cloudsimulation/zones# dladm create-simnet singaporesw1_255

root@cloudinabox:/opt/cloudsimulation/zones# dladm create-simnet londonsw1_1
root@cloudinabox:/opt/cloudsimulation/zones# dladm create-simnet londonsw1_2
root@cloudinabox:/opt/cloudsimulation/zones# dladm create-simnet hamburgsw1_1
root@cloudinabox:/opt/cloudsimulation/zones# dladm create-simnet singaporesw1_1
root@cloudinabox:/opt/cloudsimulation/zones# dladm create-simnet newyorksw1_1
root@cloudinabox:/opt/cloudsimulation/zones# dladm create-simnet sanfranciscosw1_1

root@cloudinabox:/opt/cloudsimulation/zones# dladm add-bridge -l londonsw1_1 -l londonsw1_2 -l londonsw1_255 london
root@cloudinabox:/opt/cloudsimulation/zones# dladm add-bridge -l hamburgsw1_1 -l hamburgsw1_255 hamburg
root@cloudinabox:/opt/cloudsimulation/zones# dladm add-bridge -l singaporesw1_1 -l singaporesw1_255 singapore
root@cloudinabox:/opt/cloudsimulation/zones# dladm add-bridge -l sanfranciscosw1_1 -l sanfranciscosw1_255 sanfrancisco
root@cloudinabox:/opt/cloudsimulation/zones# dladm add-bridge -l newyorksw1_1 -l newyorksw1_255 newyork

root@cloudinabox:/opt/cloudsimulation/zones# dladm create-simnet londonrouter0
root@cloudinabox:/opt/cloudsimulation/zones# dladm create-simnet londonrouter1
root@cloudinabox:/opt/cloudsimulation/zones# dladm create-simnet hamburgrouter0
root@cloudinabox:/opt/cloudsimulation/zones# dladm create-simnet hamburgrouter1
root@cloudinabox:/opt/cloudsimulation/zones# dladm create-simnet hamburgrouter2
root@cloudinabox:/opt/cloudsimulation/zones# dladm create-simnet hamburgrouter3
root@cloudinabox:/opt/cloudsimulation/zones# dladm create-simnet newyorkrouter0
root@cloudinabox:/opt/cloudsimulation/zones# dladm create-simnet newyorkrouter1
root@cloudinabox:/opt/cloudsimulation/zones# dladm create-simnet sinrouter0
root@cloudinabox:/opt/cloudsimulation/zones# dladm create-simnet sinrouter1
root@cloudinabox:/opt/cloudsimulation/zones# dladm create-simnet sinrouter2
root@cloudinabox:/opt/cloudsimulation/zones# dladm create-simnet sforouter0
root@cloudinabox:/opt/cloudsimulation/zones# dladm create-simnet sforouter1
root@cloudinabox:/opt/cloudsimulation/zones# dladm create-simnet sforouter2
root@cloudinabox:/opt/cloudsimulation/zones# dladm create-simnet sforouter3

root@cloudinabox:/opt/cloudsimulation/zones# dladm create-simnet londonsrv1
root@cloudinabox:/opt/cloudsimulation/zones# dladm create-simnet londonsrv2
root@cloudinabox:/opt/cloudsimulation/zones# dladm create-simnet hamburgsrv1
root@cloudinabox:/opt/cloudsimulation/zones# dladm create-simnet singaporesrv1
root@cloudinabox:/opt/cloudsimulation/zones# dladm create-simnet sanfranciscosrv1
root@cloudinabox:/opt/cloudsimulation/zones# dladm create-simnet newyorksrv1

root@cloudinabox:/opt/cloudsimulation/zones# dladm modify-simnet -p londonsw1_255 londonrouter0
root@cloudinabox:/opt/cloudsimulation/zones# dladm modify-simnet -p hamburgsw1_255 hamburgrouter0
root@cloudinabox:/opt/cloudsimulation/zones# dladm modify-simnet -p singaporesw1_255 sinrouter0
root@cloudinabox:/opt/cloudsimulation/zones# dladm modify-simnet -p newyorksw1_255 newyorkrouter0
root@cloudinabox:/opt/cloudsimulation/zones# dladm modify-simnet -p sanfranciscosw1_255 sforouter0
root@cloudinabox:/opt/cloudsimulation/zones# dladm modify-simnet -p hamburgrouter1 londonrouter1
root@cloudinabox:/opt/cloudsimulation/zones# dladm modify-simnet -p sforouter1 hamburgrouter2
root@cloudinabox:/opt/cloudsimulation/zones# dladm modify-simnet -p sinrouter1 hamburgrouter3
root@cloudinabox:/opt/cloudsimulation/zones# dladm modify-simnet -p sforouter3 sinrouter2
root@cloudinabox:/opt/cloudsimulation/zones# dladm modify-simnet -p newyorkrouter1 sforouter2

root@cloudinabox:/opt/cloudsimulation/zones# dladm modify-simnet -p hamburgsw1_1 hamburgsrv1
root@cloudinabox:/opt/cloudsimulation/zones# dladm modify-simnet -p singaporesw1_1 singaporesrv1
root@cloudinabox:/opt/cloudsimulation/zones# dladm modify-simnet -p newyorksw1_1 newyorksrv1
root@cloudinabox:/opt/cloudsimulation/zones# dladm modify-simnet -p londonsw1_1 londonsrv1
root@cloudinabox:/opt/cloudsimulation/zones# dladm modify-simnet -p londonsw1_2 londonsrv2
root@cloudinabox:/opt/cloudsimulation/zones# dladm modify-simnet -p sanfranciscosw1_1 sanfranciscosrv1

root@cloudinabox:/home/jmoekamp# dladm show-link
LINK                CLASS     MTU    STATE    OVER
net1                phys      1500   unknown  --
net2                phys      1500   up       --
net0                phys      1500   unknown  --
london0             bridge    1500   up       londonsw1_1 londonsw1_2 londonsw1_255
hamburg0            bridge    1500   up       hamburgsw1_1 hamburgsw1_255
singapore0          bridge    1500   up       singaporesw1_1 singaporesw1_255
newyork0            bridge    1500   up       newyorksw1_1 newyorksw1_255
sanfrancisco0       bridge    1500   up       sanfranciscosw1_1 sanfranciscosw1_255
londonsw1_255       simnet    1500   up       londonrouter0
sanfranciscosw1_255 simnet    1500   up       sforouter0
newyorksw1_255      simnet    1500   up       newyorkrouter0
hamburgsw1_255      simnet    1500   up       hamburgrouter0
singaporesw1_255    simnet    1500   up       sinrouter0
londonsw1_1         simnet    1500   up       londonsrv1
londonsw1_2         simnet    1500   up       londonsrv2
hamburgsw1_1        simnet    1500   up       hamburgsrv1
singaporesw1_1      simnet    1500   up       singaporesrv1
newyorksw1_1        simnet    1500   up       newyorksrv1
sanfranciscosw1_1   simnet    1500   up       sanfranciscosrv1
londonrouter0       simnet    1500   up       londonsw1_255
londonrouter1       simnet    1500   up       hamburgrouter1
hamburgrouter0      simnet    1500   up       hamburgsw1_255
hamburgrouter1      simnet    1500   up       londonrouter1
hamburgrouter2      simnet    1500   up       sforouter1
hamburgrouter3      simnet    1500   up       sinrouter1
newyorkrouter0      simnet    1500   up       newyorksw1_255
newyorkrouter1      simnet    1500   up       sforouter2
sinrouter0          simnet    1500   up       singaporesw1_255
sinrouter1          simnet    1500   up       hamburgrouter3
sinrouter2          simnet    1500   up       sforouter3
sforouter0          simnet    1500   up       sanfranciscosw1_255
sforouter1          simnet    1500   up       hamburgrouter2
sforouter2          simnet    1500   up       newyorkrouter1
sforouter3          simnet    1500   up       sinrouter2
londonsrv1          simnet    1500   up       londonsw1_1
londonsrv2          simnet    1500   up       londonsw1_2
hamburgsrv1         simnet    1500   up       hamburgsw1_1
singaporesrv1       simnet    1500   up       singaporesw1_1
sanfranciscosrv1    simnet    1500   up       sanfranciscosw1_1
newyorksrv1         simnet    1500   up       newyorksw1_1

Zone Creation

mkdir -p /opt/cloudsimulation/zones
zfs create rpool/zones
zfs set mountpoint=/zones rpool/zones

• /opt/cloudsimulation/zones/templateserver

  create -b
  set zonepath=/zones/templateserver
  set brand=solaris
  set autoboot=false
  set ip-type=exclusive

• /opt/cloudsimulation/zones/londonrouter:

  create -b
  set zonepath=/zones/londonrouter
  set brand=solaris
  set autoboot=false
  set ip-type=exclusive
  add net
  set configure-allowed-address=true
  set physical=londonrouter0
  end
  add net
  set configure-allowed-address=true
  set physical=londonrouter1
  end

• /opt/cloudsimulation/zones/hamburgrouter

  create -b
  set zonepath=/zones/hamburgrouter
  set brand=solaris
  set autoboot=false
  set ip-type=exclusive
  add net
  set configure-allowed-address=true
  set physical=hamburgrouter0
  end
  add net
  set configure-allowed-address=true
  set physical=hamburgrouter1
  end
  add net
  set configure-allowed-address=true
  set physical=hamburgrouter2
  end
  add net
  set configure-allowed-address=true
  set physical=hamburgrouter3
  end

• /opt/cloudsimulation/zones/singaporerouter:

  create -b
  set zonepath=/zones/singaporerouter
  set brand=solaris
  set autoboot=false
  set ip-type=exclusive
  add net
  set configure-allowed-address=true
  set physical=sinrouter0
  end
  add net
  set configure-allowed-address=true
  set physical=sinrouter1
  end
  add net
  set configure-allowed-address=true
  set physical=sinrouter2
  end

• /opt/cloudsimulation/zones/sanfranciscorouter:

  create -b
  set zonepath=/zones/sanfranciscorouter
  set brand=solaris
  set autoboot=false
  set ip-type=exclusive
  add net
  set configure-allowed-address=true
  set physical=sforouter0
  end
  add net
  set configure-allowed-address=true
  set physical=sforouter1
  end
  add net
  set configure-allowed-address=true
  set physical=sforouter2
  end
  add net
  set configure-allowed-address=true
  set physical=sforouter3
  end

• /opt/cloudsimulation/zones/newyorkrouter:

  create -b
  set zonepath=/zones/newyorkrouter
  set brand=solaris
  set autoboot=false
  set ip-type=exclusive
  add net
  set configure-allowed-address=true
  set physical=newyorkrouter0
  end
  add net
  set configure-allowed-address=true
  set physical=newyorkrouter1
  end

root@cloudinabox:/opt/cloudsimulation/zones# zonecfg -z templateserver -f	/opt/cloudsimulation/zones/templateserver 
root@cloudinabox:/opt/cloudsimulation/zones# zonecfg -z templaterouter -f /opt/cloudsimulation/zones/templaterouter
root@cloudinabox:/opt/cloudsimulation/zones# zonecfg -z londonrouter	 -f /opt/cloudsimulation/zones/londonrouter 
root@cloudinabox:/opt/cloudsimulation/zones# zonecfg -z singaporerouter -f /opt/cloudsimulation/zones/singaporerouter 
root@cloudinabox:/opt/cloudsimulation/zones# zonecfg -z hamburgrouter	 -f /opt/cloudsimulation/zones/hamburgrouter 
root@cloudinabox:/opt/cloudsimulation/zones# zonecfg -z sanfranciscorouter -f /opt/cloudsimulation/zones/sanfranciscorouter
root@cloudinabox:/opt/cloudsimulation/zones# zonecfg -z newyorkrouter -f /opt/cloudsimulation/zones/newyorkrouter

root@cloudinabox:/opt/cloudsimulation/zones# zoneadm -z templateserver install
A ZFS file system has been created for this zone.
Progress being logged to /var/log/zones/zoneadm.20111217T184237Z.templateserver.install
       Image: Preparing at /zones/templateserver/root

 Install Log: /system/volatile/install.4469/install_log
 AI Manifest: /tmp/manifest.xml.oBayTi
  SC Profile: /usr/share/auto_install/sc_profiles/enable_sci.xml
    Zonename: templateserver
Installation: Starting ...

              Creating IPS image
              Installing packages from:
                  solaris
                      origin:  http://pkg.oracle.com/solaris/release/
DOWNLOAD                                  PKGS       FILES    XFER (MB)
Completed                              167/167 32062/32062  175.8/175.8

PHASE                                        ACTIONS
Install Phase                            44313/44313 

PHASE                                          ITEMS
Package State Update Phase                   167/167 
Image State Update Phase                         2/2 
Installation: Succeeded

        Note: Man pages can be obtained by installing pkg:/system/manual

 done.

        Done: Installation completed in 1423,641 seconds.


  Next Steps: Boot the zone, then log into the zone console (zlogin -C)

              to complete the configuration process.

Log saved in non-global zone as /zones/templateserver/root/var/log/zones/zoneadm.20111217T184237Z.templateserver.install

root@cloudinabox:/opt/cloudsimulation/zones# sysconfig create-profile -o template.xml

root@cloudinabox:/opt/cloudsimulation/zones# cat template.xml 
<!DOCTYPE service_bundle SYSTEM "/usr/share/lib/xml/dtd/service_bundle.dtd.1">
<service_bundle type="profile" name="sysconfig">
  <service version="1" type="service" name="system/config-user">
    <instance enabled="true" name="default">
      <property_group type="application" name="root_account">
        <propval type="astring" name="login" value="root"/>
        <propval type="astring" name="password" value="$5$35worB11$/EeCnO5t2zOHhasRQeWeVyWuGLFFUFLQGmOhKPX82m2"/>
        <propval type="astring" name="type" value="role"/>
      </property_group>
      <property_group type="application" name="user_account">
        <propval type="astring" name="login" value="radmin"/>
        <propval type="astring" name="password" value="$5$XztZ799F$GVL48echivvJcPl.BRcVvnn3/M8Z7L6LhmyVPP04J/2"/>
        <propval type="astring" name="type" value="normal"/>
        <propval type="astring" name="description" value="routeradm"/>
        <propval type="count" name="gid" value="10"/>
        <propval type="astring" name="shell" value="/usr/bin/bash"/>
        <propval type="astring" name="roles" value="root"/>
        <propval type="astring" name="profiles" value="System Administrator"/>
        <propval type="astring" name="sudoers" value="ALL=(ALL) ALL"/>
      </property_group>
    </instance>
  </service>
  <service version="1" type="service" name="system/timezone">
    <instance enabled="true" name="default">
      <property_group type="application" name="timezone">
        <propval type="astring" name="localtime" value="UTC"/>
      </property_group>
    </instance>
  </service>
  <service version="1" type="service" name="system/environment">
    <instance enabled="true" name="init">
      <property_group type="application" name="environment">
        <propval type="astring" name="LANG" value="C"/>
      </property_group>
    </instance>
  </service>
  <service version="1" type="service" name="system/identity">
    <instance enabled="true" name="node">
      <property_group type="application" name="config">
        <propval type="astring" name="nodename" value="jamphfhn"/>
      </property_group>
    </instance>
  </service>
  <service version="1" type="service" name="system/keymap">
    <instance enabled="true" name="default">
      <property_group type="system" name="keymap">
        <propval type="astring" name="layout" value="German"/>
      </property_group>
    </instance>
  </service>
  <service version="1" type="service" name="system/console-login">
    <instance enabled="true" name="default">
      <property_group type="application" name="ttymon">
        <propval type="astring" name="terminal_type" value="sun-color"/>
      </property_group>
    </instance>
  </service>
  <service version="1" type="service" name="network/physical">
    <instance enabled="true" name="default">
      <property_group type="application" name="netcfg"/>
    </instance>
  </service>
</service_bundle>

root@cloudinabox:/opt/cloudsimulation/zones# cat template.xml | sed s/jamphfhn/templaterouter/ > templaterouter.xml

root@cloudinabox:/opt/cloudsimulation/zones# zoneadm -z templaterouter clone -c /opt/cloudsimulation/zones/templaterouter.xml templateserver
A ZFS file system has been created for this zone.
Progress being logged to /var/log/zones/zoneadm.20111217T193101Z.templaterouter.clone
Log saved in non-global zone as /zones/templaterouter/root/var/log/zones/zoneadm.20111217T193101Z.templaterouter.clone

root@cloudinabox:/opt/cloudsimulation/zones# zoneadm -z templaterouter boot
root@cloudinabox:/opt/cloudsimulation/zones# zlogin -C templaterouter                
[Connected to zone 'templaterouter' console]

Hostname: unknown
Hostname: templaterouter

templaterouter console login: radmin
Password: 
Oracle Corporation      SunOS 5.11      11.0    November 2011
radmin@templaterouter:~$

radmin@templaterouter:~$ sudo bash
Password: 
Dec 17 19:36:33 templaterouter sudo:   radmin : TTY=console ; PWD=/home/radmin ; USER=root ; COMMAND=/usr/bin/bash
root@templaterouter:/home/radmin#

root@templaterouter:/home/radmin# pkg install pkg://solaris/network/telnet
           Packages to install:  1
       Create boot environment: No
Create backup boot environment: No

DOWNLOAD                                  PKGS       FILES    XFER (MB)
Completed                                  1/1         8/8      0.1/0.1

PHASE                                        ACTIONS
Install Phase                                  22/22 

PHASE                                          ITEMS
Package State Update Phase                       1/1 
Image State Update Phase                         2/2 

root@templaterouter:/home/radmin# pkg install quagga                      
           Packages to install:  1
       Create boot environment: No
Create backup boot environment: No
            Services to change:  3

DOWNLOAD                                  PKGS       FILES    XFER (MB)
Completed                                  1/1       89/89      2.7/2.7

PHASE                                        ACTIONS
Install Phase                                132/132 

PHASE                                          ITEMS
Package State Update Phase                       1/1 
Image State Update Phase                         2/2 
Loading smf(5) service descriptions: 2/2

root@templaterouter:/home/radmin# routeadm -e ipv4-forwarding

root@templaterouter:/home/radmin# routeadm -e ipv4-routing

root@templaterouter:/home/radmin# routeadm -m zebra:quagga vty_port="2602"
root@templaterouter:/home/radmin# routeadm -m ospf:quagga vty_port="2601"
root@templaterouter:/home/radmin# routeadm -m zebra:quagga vty_address="127.0.0.1"
root@templaterouter:/home/radmin# routeadm -m ospf:quagga vty_address="127.0.0.1"

root@templaterouter:/home/radmin# routeadm -s routing-svcs=ospf:quagga -e ipv4-routing

root@templaterouter:/home/radmin# routeadm -u

root@cloudinabox:/home/jmoekamp# zoneadm -z templaterouter halt

• London
• /opt/cloudsimulation/zones/zebra.london.conf

  hostname londonrouter 
  password nomnomnom 
  enable password nonnomnom 
  log file /var/adm/quagga/zebra.log
  line vty

• /opt/cloudsimulation/zones/ospfd.london.conf

  hostname londonrouter
  password nomnomnom
  enable password nonnomnom
  log file /var/adm/quagga/ospf.log
  !
  !
  !
  interface lo0
  !
  interface londonrouter0
  !
  interface londonrouter1
  !
  router ospf
   redistribute connected
   network 10.1.1.0/24 area 0.0.0.0
  !
  line vty
  !
  



• Hamburg


• /opt/cloudsimulation/zones/zebra.hamburg.conf

  hostname hamburgrouter 
  password nomnomnom 
  enable password nonnomnom 
  log file /var/adm/quagga/zebra.log
  line vty

• /opt/cloudsimulation/zones/ospfd.hamburg.conf

  hostname hamburgrouter
  password nomnomnom
  enable password nonnomnom
  log file /var/adm/quagga/ospf.log
  !
  !
  !
  interface lo0
  !
  interface hamburgrouter0
  !
  interface hamburgrouter1
  !
  interface hamburgrouter2
  !
  interface hamburgrouter3
  !
  router ospf
   redistribute connected
   network 10.1.1.0/24 area 0.0.0.0
   network 10.1.2.0/24 area 0.0.0.0
   network 10.1.3.0/24 area 0.0.0.0 
  !
  line vty
  !



• Singapore


• /opt/cloudsimulation/zones/zebra.singapore.conf

  !
  ! Zebra configuration saved from vty
  !   2011/12/12 20:20:13
  !
  hostname sinrouter
  password nomnomnom
  enable password nomnomnom
  log file /var/adm/quagga/zebra.log
  !
  interface lo0
  !
  line vty
  !

• /opt/cloudsimulation/zones/ospfd.singapore.conf

  password nomnomnom
  enable password nonnomnom
  log file /var/adm/quagga/ospf.log
  !
  interface lo0
  !
  interface sinrouter0
  !
  interface sinrouter1
  ! 
  interface sinrouter2 
  !
  router ospf
   redistribute connected
   network 10.1.2.0/24 area 0.0.0.0
   network 10.1.4.0/24 area 0.0.0.0
  !
  line vty
  !



• San Francisco


• /opt/cloudsimulation/zones/zebra.sanfrancisco.conf

  hostname sforouter 
  password nomnomnom 
  enable password nonnomnom 
  log file /var/adm/quagga/zebra.log
  line vty

• /opt/cloudsimulation/zones/ospfd.sanfrancisco.conf

  !
  ! Zebra configuration saved from vty
  !   2011/12/11 04:30:44
  !
  hostname sforouter
  password nomnomnom
  enable password nonnomnom
  log file /var/adm/quagga/ospf.log
  !
  !
  !
  interface lo0
  !
  interface sforouter0
  !
  interface sforouter1
  !
  interface sforouter2
  !
  interface sforouter3
  !
  router ospf
   redistribute connected
   network 10.1.5.0/24 area 0.0.0.0
   network 10.1.4.0/24 area 0.0.0.0
   network 10.1.3.0/24 area 0.0.0.0
  !
  line vty
  !



• New York


• /opt/cloudsimulation/zones/zebra.newyork.conf

  hostname newyorkrouter 
  password nomnomnom 
  enable password nonnomnom 
  log file /var/adm/quagga/zebra.log
  line vty

• /opt/cloudsimulation/zones/ospfd.newyork.conf

  !
  ! Zebra configuration saved from vty
  !   2011/12/11 04:30:44
  !
  hostname newyorkrouter
  password nomnomnom
  enable password nonnomnom
  log file /var/adm/quagga/ospf.log
  !
  !
  !
  interface lo0
  !
  interface newyorkrouter0
  !
  interface newyorkrouter1
  !
  router ospf
   redistribute connected
   network 10.1.5.0/24 area 0.0.0.0
  !
  line vty
  !

root@cloudinabox:/opt/cloudsimulation/zones# cat template.xml | sed s/jamphfhn/londonrouter/ > londonrouter.xml
root@cloudinabox:/opt/cloudsimulation/zones# zoneadm -z londonrouter clone -c /opt/cloudsimulation/zones/londonrouter.xml templaterouter
A ZFS file system has been created for this zone.
Progress being logged to /var/log/zones/zoneadm.20111217T205338Z.londonrouter.clone
Log saved in non-global zone as /zones/londonrouter/root/var/log/zones/zoneadm.20111217T205338Z.londonrouter.clone
root@cloudinabox:/opt/cloudsimulation/zones# cp zebra.london.conf /zones/londonrouter/root/etc/quagga/zebra.conf
root@cloudinabox:/opt/cloudsimulation/zones# cp ospfd.london.conf /zones/londonrouter/root/etc/quagga/ospfd.conf
root@cloudinabox:/opt/cloudsimulation/zones# zoneadm -z londonrouter boot

root@cloudinabox:/opt/cloudsimulation/zones# zlogin -C londonrouter
[Connected to zone 'londonrouter' console]

londonrouter console login: radmin
Password: 
Last login: Sat Dec 17 19:35:39 on console
Oracle Corporation      SunOS 5.11      11.0    November 2011
radmin@londonrouter:~$ sudo bash
Password: 
Dec 17 20:57:58 londonrouter sudo:   radmin : TTY=console ; PWD=/home/radmin ; USER=root ; COMMAND=/usr/bin/bash
root@londonrouter:/home/radmin# 

# ipadm create-ip londonrouter0
# ipadm create-ip londonrouter1
# ipadm create-addr -T static -a 10.0.10.254/24 londonrouter0/v4
# ipadm create-addr -T static -a 10.1.1.254/24 londonrouter1/v4
# svcadm restart zebra
# svcadm restart ospf

root@londonrouter:/home/radmin# dladm show-link
LINK                CLASS     MTU    STATE    OVER
londonrouter0       simnet    1500   up       ?
londonrouter1       simnet    1500   up       ?

root@londonrouter:/home/radmin# ipadm show-addr
ADDROBJ           TYPE     STATE        ADDR
lo0/v4            static   ok           127.0.0.1/8
londonrouter0/v4  static   ok           10.0.10.254/24
londonrouter1/v4  static   ok           10.1.1.254/24
lo0/v6            static   ok           ::1/128

root@londonrouter:/home/radmin# telnet localhost 2601
Trying ::1...
telnet: connect to address ::1: Connection refused
Trying 127.0.0.1...
Connected to londonrouter.
Escape character is '^]'.

Hello, this is Quagga (version 0.99.8).
Copyright 1996-2005 Kunihiro Ishiguro, et al


User Access Verification

Password: 

londonrouter> show ip ospf interface 
lo0 is up
  ifindex 1, MTU 8232 bytes, BW 0 Kbit 
  OSPF not enabled on this interface
londonrouter0 is up
  ifindex 3, MTU 1500 bytes, BW 0 Kbit 
  OSPF not enabled on this interface
londonrouter1 is up
  ifindex 2, MTU 1500 bytes, BW 0 Kbit 
  Internet Address 10.1.1.254/24, Area 0.0.0.0
  MTU mismatch detection:enabled
  Router ID 10.1.1.254, Network Type BROADCAST, Cost: 10
  Transmit Delay is 1 sec, State DR, Priority 1
  Designated Router (ID) 10.1.1.254, Interface Address 10.1.1.254
  No backup designated router on this network
  Multicast group memberships: OSPFAllRouters OSPFDesignatedRouters
  Timer intervals configured, Hello 10s, Dead 40s, Wait 40s, Retransmit 5
    Hello due in 2.263s
  Neighbor Count is 0, Adjacent neighbor count is 0
londonrouter> exit
Connection to londonrouter closed by foreign host.

root@londonrouter:/home/radmin# telnet localhost 2602
Trying ::1...
telnet: connect to address ::1: Connection refused
Trying 127.0.0.1...
Connected to londonrouter.
Escape character is '^]'.

Hello, this is Quagga (version 0.99.8).
Copyright 1996-2005 Kunihiro Ishiguro, et al.


User Access Verification

Password: 
londonrouter> show ip route      
Codes: K - kernel route, C - connected, S - static, R - RIP, O - OSPF,
       I - ISIS, B - BGP, > - selected route, * - FIB route

C>* 10.0.10.0/24 is directly connected, londonrouter0
O   10.1.1.0/24 [110/10] is directly connected, londonrouter1, 00:04:28
C>* 10.1.1.0/24 is directly connected, londonrouter1
C>* 127.0.0.0/8 is directly connected, lo0
londonrouter>exit

root@londonrouter:/home/radmin# ~.
[Connection to zone 'londonrouter' console closed]

root@cloudinabox:/opt/cloudsimulation/zones# cat template.xml | sed s/jamphfhn/hamburgrouter/ > hamburgrouter.xml

root@cloudinabox:/opt/cloudsimulation/zones# zoneadm -z hamburgrouter clone -c /opt/cloudsimulation/zones/hamburgrouter.xml templaterouter
A ZFS file system has been created for this zone.
Progress being logged to /var/log/zones/zoneadm.20111217T212009Z.hamburgrouter.clone
Log saved in non-global zone as /zones/hamburgrouter/root/var/log/zones/zoneadm.20111217T212009Z.hamburgrouter.clone

root@cloudinabox:/opt/cloudsimulation/zones# cp ospfd.hamburg.conf /zones/hamburgrouter/root/etc/quagga/ospfd.conf
root@cloudinabox:/opt/cloudsimulation/zones# cp zebra.hamburg.conf /zones/hamburgrouter/root/etc/quagga/zebra.conf
root@cloudinabox:/opt/cloudsimulation/zones# zoneadm -z hamburgrouter boot

root@cloudinabox:/opt/cloudsimulation/zones# zlogin -C hamburgrouter
[Connected to zone 'hamburgrouter' console]


Hostname: hamburgrouter


hamburgrouter console login: radmin
Password: 
Last login: Sat Dec 17 19:35:39 on console
Oracle Corporation      SunOS 5.11      11.0    November 2011
radmin@hamburgrouter:~$ sudo bash
Password: 
Dec 17 21:23:45 hamburgrouter sudo:   radmin : TTY=console ; PWD=/home/radmin ; USER=root ; COMMAND=/usr/bin/bash
root@hamburgrouter:/home/radmin#


root@hamburgrouter:/home/radmin# ipadm create-ip hamburgrouter0
root@hamburgrouter:/home/radmin# ipadm create-ip hamburgrouter1
root@hamburgrouter:/home/radmin# ipadm create-ip hamburgrouter2
root@hamburgrouter:/home/radmin# ipadm create-ip hamburgrouter3
root@hamburgrouter:/home/radmin# ipadm create-addr -T static -a 10.0.11.254/24 hamburgrouter0/v4
root@hamburgrouter:/home/radmin# ipadm create-addr -T static -a 10.1.1.1/24 hamburgrouter1/v4
root@hamburgrouter:/home/radmin# ipadm create-addr -T static -a 10.1.3.1/24 hamburgrouter2/v4
root@hamburgrouter:/home/radmin# ipadm create-addr -T static -a 10.1.2.1/24 hamburgrouter3/v4
root@hamburgrouter:/home/radmin# svcadm restart ospf

root@hamburgrouter:/home/radmin# ipadm show-addr
ADDROBJ           TYPE     STATE        ADDR
lo0/v4            static   ok           127.0.0.1/8
hamburgrouter0/v4 static   ok           10.0.11.254/24
hamburgrouter1/v4 static   ok           10.1.1.1/24
hamburgrouter2/v4 static   ok           10.1.3.1/24
hamburgrouter3/v4 static   ok           10.1.2.1/24
lo0/v6            static   ok           ::1/128

root@hamburgrouter:/home/radmin# netstat -nr

Routing Table: IPv4
  Destination           Gateway           Flags  Ref     Use     Interface 
-------------------- -------------------- ----- ----- ---------- --------- 
10.0.10.0            10.1.1.254           UG        1          0           
10.0.11.0            10.0.11.254          U         2          0 hamburgrouter0 
10.1.1.0             10.1.1.1             U         3          9 hamburgrouter1 
10.1.2.0             10.1.2.1             U         2          0 hamburgrouter3 
10.1.3.0             10.1.3.1             U         2          0 hamburgrouter2 
127.0.0.1            127.0.0.1            UH        2          0 lo0       

Routing Table: IPv6
  Destination/Mask            Gateway                   Flags Ref   Use    If   
--------------------------- --------------------------- ----- --- ------- ----- 
::1                         ::1                         UH      2       0 lo0   
root@hamburgrouter:/home/radmin#

root@hamburgrouter:/home/radmin# telnet localhost 2602
Trying ::1...
telnet: connect to address ::1: Connection refused
Trying 127.0.0.1...
Connected to hamburgrouter.
Escape character is '^]'.

Hello, this is Quagga (version 0.99.8).
Copyright 1996-2005 Kunihiro Ishiguro, et al.


User Access Verification

Password: 
hamburgrouter> show ip route
Codes: K - kernel route, C - connected, S - static, R - RIP, O - OSPF,
       I - ISIS, B - BGP, > - selected route, * - FIB route

O>* 10.0.10.0/24 [110/20] via 10.1.1.254, hamburgrouter1, 00:01:04
C>* 10.0.11.0/24 is directly connected, hamburgrouter0
O   10.1.1.0/24 [110/10] is directly connected, hamburgrouter1, 00:01:09
C>* 10.1.1.0/24 is directly connected, hamburgrouter1
O   10.1.2.0/24 [110/10] is directly connected, hamburgrouter3, 00:01:09
C>* 10.1.2.0/24 is directly connected, hamburgrouter3
O   10.1.3.0/24 [110/10] is directly connected, hamburgrouter2, 00:01:09
C>* 10.1.3.0/24 is directly connected, hamburgrouter2
C>* 127.0.0.0/8 is directly connected, lo0
hamburgrouter> exit
Connection to hamburgrouter closed by foreign host.
root@hamburgrouter:/home/radmin#      
root@hamburgrouter:/home/radmin# ~.
[Connection to zone 'hamburgrouter' console closed]

root@cloudinabox:/opt/cloudsimulation/zones# cat template.xml | sed s/jamphfhn/singaporerouter/ > singaporerouter.xml
root@cloudinabox:/opt/cloudsimulation/zones# zoneadm -z singaporerouter clone -c /opt/cloudsimulation/zones/singaporerouter.xml templaterouter
A ZFS file system has been created for this zone.
Progress being logged to /var/log/zones/zoneadm.20111217T223455Z.singaporerouter.clone
Log saved in non-global zone as /zones/singaporerouter/root/var/log/zones/zoneadm.20111217T223455Z.singaporerouter.clone

root@cloudinabox:/opt/cloudsimulation/zones# cp ospfd.singapore.conf /zones/singaporerouter/root/etc/quagga/ospfd.conf
root@cloudinabox:/opt/cloudsimulation/zones# cp zebra.singapore.conf /zones/singaporerouter/root/etc/quagga/zebra.conf
root@cloudinabox:/opt/cloudsimulation/zones# zoneadm -z singaporerouter boot

root@cloudinabox:/opt/cloudsimulation/zones# zlogin -C singaporerouter
[Connected to zone 'singaporerouter' console]
 
singaporerouter console login: radmin
Password: 
Last login: Sat Dec 17 19:35:39 on console
Oracle Corporation      SunOS 5.11      11.0    November 2011
radmin@singaporerouter:~$ sudo bash
Password: 
Dec 17 22:37:58 singaporerouter sudo:   radmin : TTY=console ; PWD=/home/radmin ; USER=root ; COMMAND=/usr/bin/bash
ipadm create-ip sinrouter0
ipadm create-ip sinrouter1
ipadm create-ip sinrouter2
ipadm create-addr -T static -a 10.0.12.254/24 sinrouter0/v4
ipadm create-addr -T static -a 10.1.2.254/24 sinrouter1/v4
ipadm create-addr -T static -a 10.1.4.1/24 sinrouter2/v4
root@singaporerouter:/home/radmin# svcadm restart zebra
root@singaporerouter:/home/radmin# svcadm restart ospf 

root@singaporerouter:/home/radmin# telnet localhost 2602
Trying ::1...
telnet: connect to address ::1: Connection refused
Trying 127.0.0.1...
Connected to singaporerouter.
Escape character is '^]'.

Hello, this is Quagga (version 0.99.8).
Copyright 1996-2005 Kunihiro Ishiguro, et al.


User Access Verification

Password: 
sinrouter> show ip route
Codes: K - kernel route, C - connected, S - static, R - RIP, O - OSPF,
       I - ISIS, B - BGP, > - selected route, * - FIB route

O>* 10.0.10.0/24 [110/20] via 10.1.2.1, sinrouter1, 00:00:34
O>* 10.0.11.0/24 [110/20] via 10.1.2.1, sinrouter1, 00:00:34
C>* 10.0.12.0/24 is directly connected, sinrouter0
O>* 10.1.1.0/24 [110/20] via 10.1.2.1, sinrouter1, 00:00:35
O   10.1.2.0/24 [110/10] is directly connected, sinrouter1, 00:00:44
C>* 10.1.2.0/24 is directly connected, sinrouter1
O>* 10.1.3.0/24 [110/20] via 10.1.2.1, sinrouter1, 00:00:35
O   10.1.4.0/24 [110/10] is directly connected, sinrouter2, 00:00:44
C>* 10.1.4.0/24 is directly connected, sinrouter2
C>* 127.0.0.0/8 is directly connected, lo0
sinrouter> exit
Connection to singaporerouter closed by foreign host.
root@singaporerouter:/home/radmin# ~.
[Connection to zone 'singaporerouter' console closed]

root@cloudinabox:/opt/cloudsimulation/zones# cat template.xml | sed s/jamphfhn/sanfranciscorouter/ > sanfranciscorouter.xml
root@cloudinabox:/opt/cloudsimulation/zones# zoneadm -z sanfranciscorouter clone -c /opt/cloudsimulation/zones/sanfranciscorouter.xml templaterouter
A ZFS file system has been created for this zone.
Progress being logged to /var/log/zones/zoneadm.20111217T224355Z.sanfranciscorouter.clone
Log saved in non-global zone as /zones/sanfranciscorouter/root/var/log/zones/zoneadm.20111217T224355Z.sanfranciscorouter.clone

root@cloudinabox:/opt/cloudsimulation/zones# cp ospfd.sanfrancisco.conf /zones/sanfranciscorouter/root/etc/quagga/ospfd.conf
root@cloudinabox:/opt/cloudsimulation/zones# cp zebra.sanfrancisco.conf /zones/sanfranciscorouter/root/etc/quagga/zebra.conf
root@cloudinabox:/opt/cloudsimulation/zones# zoneadm -z sanfranciscorouter boot

root@cloudinabox:/opt/cloudsimulation/zones# zlogin -C sanfranciscorouter
[Connected to zone 'sanfranciscorouter' console]

sanfranciscorouter console login: radmin
Password: 
Last login: Sat Dec 17 19:35:39 on console
Oracle Corporation      SunOS 5.11      11.0    November 2011
radmin@sanfranciscorouter:~$ sudo bash
Password: 
Dec 17 22:46:49 sanfranciscorouter sudo:   radmin : TTY=console ; PWD=/home/radmin ; USER=root ; COMMAND=/usr/bin/bash

ipadm create-ip sforouter0
ipadm create-ip sforouter1
ipadm create-ip sforouter2
ipadm create-ip sforouter3
ipadm create-addr -T static -a 10.0.13.254/24 sforouter0/v4
ipadm create-addr -T static -a 10.1.3.254/24 sforouter1/v4
ipadm create-addr -T static -a 10.1.5.1/24 sforouter2/v4
ipadm create-addr -T static -a 10.1.4.254/24 sforouter3/v4

root@sanfranciscorouter:/home/radmin# telnet localhost 2602
Trying ::1...
telnet: connect to address ::1: Connection refused
Trying 127.0.0.1...
Connected to sanfranciscorouter.
Escape character is '^]'.

Hello, this is Quagga (version 0.99.8).
Copyright 1996-2005 Kunihiro Ishiguro, et al.


User Access Verification

Password: 
sforouter> show ip route 
Codes: K - kernel route, C - connected, S - static, R - RIP, O - OSPF,
       I - ISIS, B - BGP, > - selected route, * - FIB route

O>* 10.0.10.0/24 [110/20] via 10.1.3.1, sforouter1, 00:00:25
O>* 10.0.11.0/24 [110/20] via 10.1.3.1, sforouter1, 00:00:25
O>* 10.0.12.0/24 [110/20] via 10.1.4.1, sforouter3, 00:00:25
C>* 10.0.13.0/24 is directly connected, sforouter0
O>* 10.1.1.0/24 [110/20] via 10.1.3.1, sforouter1, 00:00:26
O>* 10.1.2.0/24 [110/20] via 10.1.3.1, sforouter1, 00:00:26
  *                      via 10.1.4.1, sforouter3, 00:00:26
O   10.1.3.0/24 [110/10] is directly connected, sforouter1, 00:00:26
C>* 10.1.3.0/24 is directly connected, sforouter1
O   10.1.4.0/24 [110/10] is directly connected, sforouter3, 00:00:35
C>* 10.1.4.0/24 is directly connected, sforouter3
O   10.1.5.0/24 [110/10] is directly connected, sforouter2, 00:00:35
C>* 10.1.5.0/24 is directly connected, sforouter2
C>* 127.0.0.0/8 is directly connected, lo0
sforouter> exit
Connection to sanfranciscorouter closed by foreign host.
root@sanfranciscorouter:/home/radmin# ~.
[Connection to zone 'sanfranciscorouter' console closed]

root@cloudinabox:/opt/cloudsimulation/zones# cat template.xml | sed s/jamphfhn/newyorkrouter/ > newyorkrouter.xml
root@cloudinabox:/opt/cloudsimulation/zones# zoneadm -z newyorkrouter clone -c /opt/cloudsimulation/zones/newyorkrouter.xml templaterouter
A ZFS file system has been created for this zone.
Progress being logged to /var/log/zones/zoneadm.20111217T225139Z.newyorkrouter.clone
Log saved in non-global zone as /zones/newyorkrouter/root/var/log/zones/zoneadm.20111217T225139Z.newyorkrouter.clone
root@cloudinabox:/opt/cloudsimulation/zones# cp ospfd.newyork.conf /zones/newyorkrouter/root/etc/quagga/ospfd.conf
root@cloudinabox:/opt/cloudsimulation/zones# cp zebra.newyork.conf /zones/newyorkrouter/root/etc/quagga/zebra.conf
root@cloudinabox:/opt/cloudsimulation/zones# zoneadm -z newyorkrouter boot

root@cloudinabox:/opt/cloudsimulation/zones# zlogin -C newyorkrouter
[Connected to zone 'newyorkrouter' console]
newyorkrouter console login: radmin
Password: 
Last login: Sat Dec 17 19:35:39 on console
Oracle Corporation      SunOS 5.11      11.0    November 2011
radmin@newyorkrouter:~$ sudo bash
Password: 
Dec 17 22:54:33 newyorkrouter sudo:   radmin : TTY=console ; PWD=/home/radmin ; USER=root ; COMMAND=/usr/bin/bash

ipadm create-ip newyorkrouter0
ipadm create-ip newyorkrouter1
ipadm create-addr -T static -a 10.0.14.0/24 newyorkrouter0/v4
ipadm create-addr -T static -a 10.1.5.254/24 newyorkrouter1/v4

root@newyorkrouter:/home/radmin# telnet localhost 2602
Trying ::1...
telnet: connect to address ::1: Connection refused
Trying 127.0.0.1...
Connected to newyorkrouter.
Escape character is '^]'.

Hello, this is Quagga (version 0.99.8).
Copyright 1996-2005 Kunihiro Ishiguro, et al.


User Access Verification

Password: 
newyorkrouter> show ip route
Codes: K - kernel route, C - connected, S - static, R - RIP, O - OSPF,
       I - ISIS, B - BGP, > - selected route, * - FIB route

O>* 10.0.10.0/24 [110/20] via 10.1.5.1, newyorkrouter1, 00:00:22
O>* 10.0.11.0/24 [110/20] via 10.1.5.1, newyorkrouter1, 00:00:22
O>* 10.0.12.0/24 [110/20] via 10.1.5.1, newyorkrouter1, 00:00:22
O>* 10.0.13.0/24 [110/20] via 10.1.5.1, newyorkrouter1, 00:00:22
C>* 10.0.14.0/24 is directly connected, newyorkrouter0
O>* 10.1.1.0/24 [110/30] via 10.1.5.1, newyorkrouter1, 00:00:23
O>* 10.1.2.0/24 [110/30] via 10.1.5.1, newyorkrouter1, 00:00:23
O>* 10.1.3.0/24 [110/20] via 10.1.5.1, newyorkrouter1, 00:00:23
O>* 10.1.4.0/24 [110/20] via 10.1.5.1, newyorkrouter1, 00:00:23
O   10.1.5.0/24 [110/10] is directly connected, newyorkrouter1, 00:00:25
C>* 10.1.5.0/24 is directly connected, newyorkrouter1
C>* 127.0.0.0/8 is directly connected, lo0
newyorkrouter> exit
Connection to newyorkrouter closed by foreign host.
root@newyorkrouter:/home/radmin# ~.
[Connection to zone 'newyorkrouter' console closed]

root@cloudinabox:/opt/cloudsimulation/zones# zlogin -C newyorkrouter
[Connected to zone 'newyorkrouter' console]

root@newyorkrouter:/home/radmin# 
root@newyorkrouter:/home/radmin# 
root@newyorkrouter:/home/radmin# ping 10.0.10.254
10.0.10.254 is alive
root@newyorkrouter:/home/radmin# traceroute 10.0.10.254
traceroute: Warning: Multiple interfaces found; using 10.1.5.254 @ newyorkrouter1
traceroute to 10.0.10.254 (10.0.10.254), 30 hops max, 40 byte packets
 1  10.1.5.1 (10.1.5.1)  0.116 ms  0.083 ms  0.038 ms
 2  10.1.3.1 (10.1.3.1)  0.072 ms  0.048 ms  0.041 ms
 3  10.0.10.254 (10.0.10.254)  0.065 ms  0.077 ms  0.047 ms

root@cloudinabox:/opt/cloudsimulation/zones# zlogin -C sanfranciscorouter
[Connected to zone 'sanfranciscorouter' console]

root@sanfranciscorouter:/home/radmin# 
root@sanfranciscorouter:/home/radmin# 
root@sanfranciscorouter:/home/radmin# ipadm disable-if sforouter1
ipadm: persistent operation not supported for disable-if
root@sanfranciscorouter:/home/radmin# ipadm disable-if -t  sforouter1
root@sanfranciscorouter:/home/radmin# ~.
bash: ~.: command not found
root@sanfranciscorouter:/home/radmin# ~.
[Connection to zone 'sanfranciscorouter' console closed]
root@cloudinabox:/opt/cloudsimulation/zones# zlogin -C newyorkrouter
[Connected to zone 'newyorkrouter' console]

root@newyorkrouter:/home/radmin# traceroute 10.0.10.254
traceroute: Warning: Multiple interfaces found; using 10.1.5.254 @ newyorkrouter1
traceroute to 10.0.10.254 (10.0.10.254), 30 hops max, 40 byte packets
 1  10.1.5.1 (10.1.5.1)  0.114 ms  0.141 ms  0.132 ms
 2  10.1.4.1 (10.1.4.1)  0.072 ms  0.046 ms  0.041 ms
 3  10.1.2.1 (10.1.2.1)  0.065 ms  0.066 ms  0.048 ms
 4  10.0.10.254 (10.0.10.254)  0.073 ms  0.068 ms  0.052 ms

root@newyorkrouter:/home/radmin# ~.
[Connection to zone 'newyorkrouter' console closed]

root@cloudinabox:/opt/cloudsimulation/zones# 
root@cloudinabox:/opt/cloudsimulation/zones# zlogin -C sanfranciscorouter
[Connected to zone 'sanfranciscorouter' console]

root@sanfranciscorouter:/home/radmin# 
root@sanfranciscorouter:/home/radmin# ipadm enable-if -t  sforouter1
root@sanfranciscorouter:/home/radmin# 
root@sanfranciscorouter:/home/radmin# ~.
[Connection to zone 'sanfranciscorouter' console closed]

root@cloudinabox:/opt/cloudsimulation/zones# 
root@cloudinabox:/opt/cloudsimulation/zones# zlogin -C newyorkrouter
[Connected to zone 'newyorkrouter' console]

root@newyorkrouter:/home/radmin# traceroute 10.0.10.254
traceroute: Warning: Multiple interfaces found; using 10.1.5.254 @ newyorkrouter1
traceroute to 10.0.10.254 (10.0.10.254), 30 hops max, 40 byte packets
 1  10.1.5.1 (10.1.5.1)  0.361 ms  0.044 ms  0.037 ms
 2  10.1.3.1 (10.1.3.1)  0.061 ms  0.046 ms  0.042 ms
 3  10.0.10.254 (10.0.10.254)  0.070 ms  0.052 ms  0.048 ms
root@newyorkrouter:/home/radmin# ~.
[Connection to zone 'newyorkrouter' console closed]

zonecfg -z londonsrv1 export
create -b
set zonepath=/zones/londonsrv1
set brand=solaris
set autoboot=false
set ip-type=exclusive
add net
set configure-allowed-address=true
set physical=londonsrv1
end

root@cloudinabox:/opt/cloudsimulation/zones# cat template.xml | sed s/jamphfhn/londonsrv1/ > londonsrv1.xml  
root@cloudinabox:/opt/cloudsimulation/zones# zonecfg -z londonsrv1 -f londonsrv1
root@cloudinabox:/opt/cloudsimulation/zones# zoneadm -z londonsrv1 clone -c /opt/cloudsimulation/zones/londonsrv1.xml templateserver
A ZFS file system has been created for this zone.
Progress being logged to /var/log/zones/zoneadm.20111218T043435Z.londonsrv1.clone
Log saved in non-global zone as /zones/londonsrv1/root/var/log/zones/zoneadm.20111218T043435Z.londonsrv1.clone
root@cloudinabox:/opt/cloudsimulation/zones# zoneadm -z londonsrv1 boot

root@cloudinabox:/opt/cloudsimulation/zones# zlogin -C londonsrv1
[Connected to zone 'londonsrv1' console]

londonsrv1 console login: radmin
Password: 
Oracle Corporation      SunOS 5.11      11.0    November 2011

radmin@londonsrv1:~$ sudo bash
Password: 
Dec 18 04:53:49 londonsrv1 sudo:   radmin : TTY=console ; PWD=/home/radmin ; USER=root ; COMMAND=/usr/bin/bash
root@londonsrv1:/home/radmin# ipadm create-ip londonsrv1
root@londonsrv1:/home/radmin# ipadm create-addr -T static -a 10.0.10.10/24 londons
rv1/v4
root@londonsrv1:/home/radmin# route -p add default 10.0.10.254
add net default: gateway 10.0.10.254
add persistent net default: gateway 10.0.10.254

root@londonsrv1:/home/radmin# ping 10.0.10.254
10.0.10.254 is alive
root@londonsrv1:/home/radmin# traceroute 10.0.13.254
traceroute to 10.0.13.254 (10.0.13.254), 30 hops max, 40 byte packets
 1  10.0.10.254 (10.0.10.254)  0.238 ms  0.051 ms  0.044 ms
 2  10.1.1.1 (10.1.1.1)  0.098 ms  0.057 ms  0.053 ms
 3  10.0.13.254 (10.0.13.254)  0.072 ms  0.059 ms  0.061 ms
root@londonsrv1:/home/radmin# ~.
[Connection to zone 'londonsrv1' console closed]

root@cloudinabox:/opt/cloudsimulation/zones# cat template.xml | sed s/jamphfhn/newyorksrv1/ > newyorksrv1.xml  
root@cloudinabox:/opt/cloudsimulation/zones# cat londonsrv1 | sed s/londonsrv1/newyorksrv1/ > newyorksrv1
root@cloudinabox:/opt/cloudsimulation/zones# vi newyorksrv1
root@cloudinabox:/opt/cloudsimulation/zones# zonecfg -z newyorksrv1 -f newyorksrv1
root@cloudinabox:/opt/cloudsimulation/zones# zoneadm -z newyorksrv1 clone -c /opt/cloudsimulation/zones/newyorksrv1.xml templateserver
A ZFS file system has been created for this zone.
Progress being logged to /var/log/zones/zoneadm.20111218T050558Z.newyorksrv1.clone
Log saved in non-global zone as /zones/newyorksrv1/root/var/log/zones/zoneadm.20111218T050558Z.newyorksrv1.clone
root@cloudinabox:/opt/cloudsimulation/zones# zoneadm -z newyorksrv1 boot

root@cloudinabox:/home/jmoekamp# zlogin -C newyorksrv1
[Connected to zone 'newyorksrv1' console]

newyorksrv1 console login: radmin
Password: 
Last login: Sun Dec 18 05:38:10 on console
Oracle Corporation      SunOS 5.11      11.0    November 2011
radmin@newyorksrv1:~$ sudo bash
Password:
root@newyorksrv1:/home/radmin# ipadm create-ip newyorksrv1
root@newyorksrv1:/home/radmin# ipadm create-addr -T static -a 10.0.14.10/24 new
yorksrv1/v4
root@newyorksrv1:/home/radmin# route -p add default 10.0.14.254
add net default: gateway 10.0.14.254
add persistent net default: gateway 10.0.14.254
root@newyorksrv1:/home/radmin# ping 10.0.14.254
10.0.14.254 is alive
root@newyorksrv1:/home/radmin# traceroute 10.0.10.10
traceroute to 10.0.10.10 (10.0.10.10), 30 hops max, 40 byte packets
 1  10.0.14.254 (10.0.14.254)  0.132 ms  0.051 ms  0.044 ms
 2  10.1.5.1 (10.1.5.1)  0.068 ms  0.085 ms  0.058 ms
 3  10.1.3.1 (10.1.3.1)  0.070 ms  0.057 ms  0.054 ms
 4  10.1.1.254 (10.1.1.254)  0.110 ms  0.063 ms  0.058 ms
 5  10.0.10.10 (10.0.10.10)  0.085 ms  0.069 ms  0.065 ms
root@newyorksrv1:/home/radmin#

Switches for Hamburg-MAN

root@cloudinabox:/home/jmoekamp# dladm create-simnet hamburgsw1_250
root@cloudinabox:/home/jmoekamp# dladm create-simnet hamburgsw1_251
root@cloudinabox:/home/jmoekamp# dladm create-simnet hamburgsw2_250
root@cloudinabox:/home/jmoekamp# dladm create-simnet hamburgsw2_251
root@cloudinabox:/home/jmoekamp# dladm create-simnet hamburgsw3_250
root@cloudinabox:/home/jmoekamp# dladm create-simnet hamburgsw3_251
root@cloudinabox:/home/jmoekamp# dladm modify-simnet -p hamburgsw2_250 hamburgsw1_251
root@cloudinabox:/home/jmoekamp# dladm modify-simnet -p hamburgsw3_250 hamburgsw2_251
root@cloudinabox:/home/jmoekamp# dladm modify-simnet -p hamburgsw1_250 hamburgsw3_251

root@cloudinabox:/home/jmoekamp# dladm create-bridge hamburgharbour
root@cloudinabox:/home/jmoekamp# dladm create-bridge hamburgairport

root@cloudinabox:/home/jmoekamp# dladm add-bridge -l hamburgsw1_250 -l hamburgsw1_251 hamburg
root@cloudinabox:/home/jmoekamp# dladm add-bridge -l hamburgsw2_250 -l hamburgsw2_251 hamburgairport
root@cloudinabox:/home/jmoekamp# dladm add-bridge -l hamburgsw3_250 -l hamburgsw3_251 hamburgharbour

root@cloudinabox:/home/jmoekamp# dladm show-bridge -l  hamburgharbour 
LINK        STATE       UPTIME  DESROOT
hamburgsw3_250 forwarding 55    32768/36:5d:1c:5d:ec:f
hamburgsw3_251 forwarding 55    32768/36:5d:1c:5d:ec:f
root@cloudinabox:/home/jmoekamp# dladm show-bridge -l  hamburgairport 
LINK        STATE       UPTIME  DESROOT
hamburgsw2_250 forwarding 63    32768/36:5d:1c:5d:ec:f
hamburgsw2_251 forwarding 63    32768/36:5d:1c:5d:ec:f
root@cloudinabox:/home/jmoekamp# dladm show-bridge -l  hamburg 
LINK        STATE       UPTIME  DESROOT
hamburgsw1_1 forwarding 51355   32768/36:5d:1c:5d:ec:f
hamburgsw1_255 forwarding 51355 32768/36:5d:1c:5d:ec:f
hamburgsw1_250 discarding 193   32768/36:5d:1c:5d:ec:f
hamburgsw1_251 forwarding 193   32768/36:5d:1c:5d:ec:f

root@cloudinabox:/home/jmoekamp# dladm modify-bridge -p 1 hamburg

root@cloudinabox:/opt/cloudsimulation/zones# dladm show-bridge -l  hamburg
LINK        STATE       UPTIME  DESROOT
hamburgsw1_1 forwarding 57351   0/da:c6:f6:e6:72:63
hamburgsw1_255 forwarding 57351 0/da:c6:f6:e6:72:63
hamburgsw1_250 forwarding 6189  0/da:c6:f6:e6:72:63
hamburgsw1_251 forwarding 6189  0/da:c6:f6:e6:72:63
root@cloudinabox:/opt/cloudsimulation/zones# dladm show-bridge -l  hamburgairport
LINK        STATE       UPTIME  DESROOT
hamburgsw2_250 forwarding 6090  0/da:c6:f6:e6:72:63
hamburgsw2_251 forwarding 6090  0/da:c6:f6:e6:72:63
root@cloudinabox:/opt/cloudsimulation/zones# dladm show-bridge -l  hamburgharbour
LINK        STATE       UPTIME  DESROOT
hamburgsw3_250 discarding 6098  0/da:c6:f6:e6:72:63
hamburgsw3_251 forwarding 6098  0/da:c6:f6:e6:72:63

roo 
    ]]>

    Wed, 28 Dec 2011 21:37:00 +0100
    http://www.c0t0d0s0.org/archives/7424-guid.html
    http://creativecommons.org/licenses/by-sa/2.5/
http://www.c0t0d0s0.org/archives/7424-Simulating-the-cloud-a-practical-example..html

    
    http://feedproxy.google.com/~r/C0t0d0s0org-Solaris/~3/3C4wGXq4qTU/7419-Simulating-that-clould.html
            English
            Solaris
            Sun/Oracle
    
    http://www.c0t0d0s0.org/archives/7419-Simulating-that-clould.html#comments
    http://www.c0t0d0s0.org/wfwcomment.php?cid=7419

    2
    http://www.c0t0d0s0.org/rss.php?version=2.0&type=comments&cid=7419
    

    nospam@example.com (Joerg Moellenkamp)
     ), but the cloud-like icon in many architectural diagrams called "Network" or "Internet" that sits between the client and the application that often resembles the "a wonder happens here" box in many architectures.



It's not new: I talked about this mid November at the DOAG conference in Nuremberg. And i've playing around with this at customers an privately for a while now.



Many customers have networks as large and as complex as the internet part of a smaller country perhaps 15 years ago. The interesting question is: How can you test your application for it's resiliency against failures in this cloud shaped icon.  How does your application react, when your network is doing its high availability magic.



And interestingly Solaris 11 can help you here. The thoughts behind this are pretty simple.


A router is a computer that runs an operating environment that is tailormade to do network stuff, but at the end it's a computer with a OS (yeah, i know, hardware offloading makes this a little bit more complex, but at the end it's that way)
A zone is a virtual operating environment.
Each zone can have it's own set of routes.
Each zone can have it's own set of firewall rules.
Each zone can have it's own set of processes.
Routing protocols are not more than processes collecting information from the network and configuring the routing table.
You can install a vast array of dynamic routing protocols on a zone.
I can have up to 8192 zones (given enough memory)
In Solaris 11 i can emulate switches (etherstubs)
I can limit bandwidths in Solaris 11 out-of-the-box with crossbow


When i'm combining all this features i can set up a vast array of zones doing nothing else taking each incoming packet on a interface, routing it on a multitude of ways between each other, and send it out on a outgoing interface. Even when the system in your environment are placed in many separate networks of your network you can still use a system with many networking cards or something called server-on-a-stick (single high-bandwidth connection to a vlan-trunking capable switch and using the switch ports as a fan-out).



So in order to emulate a complex corporate network, all i have to do is configuring a lot of etherstubs, configuring many vnics, replicate the physical bandwidths with the maxbw setting on the vnics, set up a lot of zones, perhaps translate the ACL of the routers into firewall rules for firewall functionality of Solaris, installting the routing daemons and configure it similar to the configuration of the routers (in regard of timeouts and so on).



Now i can test, how my applications react, when the network starts to converge against a new topology because of the failures of some lines. I can test, to which topology my network will  converge after an line outage (which is nothing more than a deny-all firewall rule). I can test the impact, when the network converges that way, that my traffic flows over a 2 MBit/s line instead of a 155 MBit/s line. For even more complex failure modes i can even use the htbx driver to introduce additional latencies, packet drop or packet reorderung as shown in this article. In essence you can emulate your complete internal network in a single box and with Zones and Crossbow in Solaris 11 it's so low overhead (at the end it is still just one kernel) that you can really emulate the reality and not a simplyfied view, as you don't have emulate via separate hardware or many independent operating system instances in virtual machines.



At the end you could simply use a single Solaris system, put it between all your test systems and use this solaris system as a emulation device for your corporate network. It's simulating the cloud-shaped icon in your architectural diagrams.

 
    ]]>

    Sat, 10 Dec 2011 09:45:38 +0100
    http://www.c0t0d0s0.org/archives/7419-guid.html
    http://creativecommons.org/licenses/by-sa/2.5/
http://www.c0t0d0s0.org/archives/7419-Simulating-that-clould.html

    
    http://feedproxy.google.com/~r/C0t0d0s0org-Solaris/~3/QYDlWWQtt90/7416-10-years-of-ZFS.html
            Solaris
    
    http://www.c0t0d0s0.org/archives/7416-10-years-of-ZFS.html#comments
    http://www.c0t0d0s0.org/wfwcomment.php?cid=7416

    8
    http://www.c0t0d0s0.org/rss.php?version=2.0&type=comments&cid=7416
    

    nospam@example.com (Joerg Moellenkamp)
    10th birthday on October 31st. So whatever you plan as a filesystem to kill ZFS ... may take a while  
    ]]>

    Wed, 02 Nov 2011 08:51:13 +0100
    http://www.c0t0d0s0.org/archives/7416-guid.html
    http://creativecommons.org/licenses/by-sa/2.5/
http://www.c0t0d0s0.org/archives/7416-10-years-of-ZFS.html

    
    http://feedproxy.google.com/~r/C0t0d0s0org-Solaris/~3/C3VWvoK1oU4/7414-Its-facepalm-time-....html
            Solaris
    
    http://www.c0t0d0s0.org/archives/7414-Its-facepalm-time-....html#comments
    http://www.c0t0d0s0.org/wfwcomment.php?cid=7414

    8
    http://www.c0t0d0s0.org/rss.php?version=2.0&type=comments&cid=7414
    

    nospam@example.com (Joerg Moellenkamp)
    


Perhaps I should explain first that c0t0d0s0.org isn’t run with Solaris, it uses this-other-unixoid-operating-system in a well-known non-commercial variant. That’s the dirty secret of c0t0d0s0.org. No technical reason for it, but webserving and mail could be done by any operating system and thus I used that operating system with ubiquitous availability at almost all providers of dedicated servers. I’m able to migrate the server from one dedicated server provider to another within 2 hours including moving the data and did this three times in the past (from 1&1 to Hetzner, and two times within Hetzner). This saved quite significant money until now and that’s the basic reason why I don’t want donations and when you do donations I would donate this money to kiva.org) 



Hetzner has reasonably priced dedicated servers and I had no problems in the past, however they have one important shortfall: No serial console in the standard product. When you need a console, you have to make a support call and they connect one. As you need it really seldom, it’s okay. As I found out later: I With this serial console I would have recognized the problem within a minute, and fixed in a second. However: The console was exactly thing that I didn’t had to my disposal at this moment.



So it was a lot harder to find out what’s happened. However i wanted my server back as soon as possible (out of personal reasons I was just able to start the recovery in the evening and as I have job to do I could only do the further stuff in the evenings as well) and thus I just reimaged the server after keeping a copy of the logfiles. I have a quite extensive backup regimen with very regular rsyncs and database replication on my server at home thus I knew I would perhaps just lose an hour of minutes of data and that was okay for me, additionally I was able to mount the disks of the non-working installation and to copy the delta of mails between the last backup and the last mail in the queue to my backup.



What had happened: At 10:something my server provider had a large power outage. The UPS didn’t take over as planed and thus a lot of servers rebooted. One of them was mine. Damned … but that’s the basic reason why I’m a fan of proper enterprise architecture and not of some singular availability features, no matter what marketing tells you. Real availability is hard work and often expensive. But:  When you really bet your business on IT, you need an architecture that is even capable to cover an UPS that proofs to be not so uninterruptible. The availability feature UPS may fail (and did fail my case) but a proper enterprise architecture keeps your service up and running. Even more important: With a proper enterprise architecture you don’t need the feature UPS for availability reasons at all because your service can survive the outage of some parts. Perhaps you want the UPS out of other reasons like “don’t want the hassle of bringing up all the systems again.”. But you don’t need it with such an architecture to keep your business running. By having a proper planed enterprise architecture with servers on two seperate sides with different power grids you may forget about the UPS because a UPS won’t help you with a prolonged power outage for example because of region-wide blackout. An outage that maybe will take out the connectivity as well as it’s not that unprobable that your local carrier has the same power problem  However my business is being an architect, not blogging and thus I didn’t have such an architecture in place for c0t0d0s0.org But that’s a different story. 



Okay: After a while my system worked again and thus I had time to find out what had happened. I knew that the system was still reacting on pings, thus I knew the kernel of  this-other-unixoid-operating-system in a well-known non-commercial variant was working. Looking into the logfiles I saw complete bootup of the kernel and some of the daemons were starting up .... like acpid for example.  However I couldn’t log into SSH. No signs in the logfiles of a ssh daemon startup. The apache was in a half-reacting state. Port 80 was open but it didn’t reacted to HTTP commands.



Out of this I concluded: The kernel and the boot configuration is okay. The bring up of the services is at least working partially, because otherwise it would start services at all. And as it reacted on the networking there must have been at least a working boot of services mandated by rcS.d, as otherwise there would be no networking. The problem must be in apache that is frozen halfway. And out of other reasons ssh isn’t started at all. There must have been a major fsckup in the startup of the services  As I had no console as explained before I needed to conclude from the leftovers what had happened.



And now was a little bit puzzled. 5-6 years ago I would have recognized this problem in an instance (because sometimes i've produced ... well ... suboptimal startup scripts) … but now today it took a while, because I didn’t felt prey for 5-6 years to such a problem. It took me a lot of more thoughts what might had happened. When you do one operating system for a living and one for hobby, you tend to project your mindset of one to the others and you don’t do justice to this other OS.



As you all may know, Solaris  ditched init.d with Solaris 10 in order to introduce SMF (not to forget the equally important features like the contract filesystem and the Fault Management Architecture). One of the nice advantages of SMF is that services that aren’t interdependent will be started in parallel without waiting for another. This has two advantages: At first the system can start up much faster, at second a service not able to start up can’t block the startup of the rest (short of services needed by all others). 



The init.d concept is a different. All services are started in sequence. The sequence is numerical and  then alphabetical. That is of course slower but more important … depending on the way you write your script a script or binary hanging or waiting for user interaction can block the startup. The variant of this-other-unixoid-operating-system is using init.d



And it’s quite easy to block a service. For example by integrating a new SSL  key and certificate. My key had a password and apache was asking for this key in order to startup. This exacly happened. Acpi started up because it was started before Apache (guess what: ACpi is before APache in an alphabetical order, and way before Ssh). This is the basic reason why you strip of the password from your key. Guess what  I did last week: I put a new key and certificate on my server and I forgot to strip the password from it.



And that exactly happened: In my version of this-other-unixoid-operating-system the ssh daemon is started after the apache daemon. When Apache waits for something you won't get SSH. Damned … it's facepalm time. Basically I felt prey to a beginners error because I’m working with an operating system that reacts totally differently on such situations. On Solaris such a situation just don’t matter at all … you get at least your ssh login and the system non-availability is just a service-non-availability you can fix within a second. However given the init.d system of  this-other-unixoid-operating-system the outcome was somewhat more problematic. However: What a dumb error on my side ...



However: The reinstallation wasn't that bad ... the system could used a reinstallation because of some tests and experiments. So it was worth the work in the evenings. And on the other side: Who had the glorious idea to start apache before ssh?



That said, this-other-unixoid-operating-system in newer variants have a different startup mechanism up upstart all the services. However: My heating control is running on a beagleboard-XM at the moment using a really current variant of this-other-unixoid-operating-system just released a few days ago. It uses this new startup mechanism. And it’s justs my unimportant personal preference, that doesn’t matter: But I don’t like it. And I have a lot of reasons for it.  It looks by far too much designed for desktop needs. However my dislike would require an article I won’t write in this blog nowadays. But as I wrote: That’s my personal opinion that doesn’t matter.



However it’s really important that this-other-unixoid-operating-system gets away from the old init.d mechanism to something more current. I think in 2011 every operating system deserves something more functional, something better than init.d … init.d is simple and well understood, however it creates classes of problems unnecessary today. Especially: In order to keep die-hard Solaris admins to fall prey to such a beginners error because such problems were parts of their distant past. And now i will start to cut holes for my eyes in the brown paper bag for my head. 
    ]]>

    Thu, 27 Oct 2011 14:04:07 +0200
    http://www.c0t0d0s0.org/archives/7414-guid.html
    http://creativecommons.org/licenses/by-sa/2.5/
http://www.c0t0d0s0.org/archives/7414-Its-facepalm-time-....html

    
    http://feedproxy.google.com/~r/C0t0d0s0org-Solaris/~3/nbaDAsO-OKs/7409-Nice-example-for-the-power-of-boot-environments.html
            English
            Solaris
            Sun/Oracle
    
    http://www.c0t0d0s0.org/archives/7409-Nice-example-for-the-power-of-boot-environments.html#comments
    http://www.c0t0d0s0.org/wfwcomment.php?cid=7409

    7
    http://www.c0t0d0s0.org/rss.php?version=2.0&type=comments&cid=7409
    

    nospam@example.com (Joerg Moellenkamp)
    


So: Please don't try this at home. Whey you try it, don't try it on any  Solaris 11 Express  installation of any value. But don't try it. I don't want to hear any story. that you've deleted your ERP system by accident because you used the wrong terminal window.  Leave that to trained professional stunt admins with the right equipment (Solaris 11 Express)  



Assume you have a system, configured with all your application, everything is running fine. So you think it would be nice to have something like a freezed state of this situation. No problem. This command will do the trick.



# beadm create rescuenet
# init 6




When you reboot your system you will see it as a new entry in the grub menu.







Okay, but boot into the old environment starting "Oracle Solaris ..."  first by selecting it in the grub menu (it should be already selected, or you used beadm activate already. Now i will drop  the atomic bomb on your installation. 

# rm --no-preserve-root -rf /
Essentially we've just nuked the installation. After a moment the system should just freeze. Reset the system and boot again via grub into the boot environment starting with "Oracle Solaris ...":







Okay ... on a normal system this would send you to the tapes. With Solaris 11: Reset the system. Boot into the boot environment "rescuenet" via selecting it in grub.







Tada! Just creating a boot environment with a single command after a config change may safe your butt later .... and btw ... this even works in zones ... they know the concepts of boot environment,too. 
    ]]>

    Tue, 04 Oct 2011 20:06:04 +0200
    http://www.c0t0d0s0.org/archives/7409-guid.html
    http://creativecommons.org/licenses/by-sa/2.5/
http://www.c0t0d0s0.org/archives/7409-Nice-example-for-the-power-of-boot-environments.html

    
    http://feedproxy.google.com/~r/C0t0d0s0org-Solaris/~3/YZjV-g36iUE/7407-How-to-activate-IPoIB-Connected-mode-in-Solaris-10-Update-9.html
            English
            Solaris
            Sun/Oracle
    
    http://www.c0t0d0s0.org/archives/7407-How-to-activate-IPoIB-Connected-mode-in-Solaris-10-Update-9.html#comments
    http://www.c0t0d0s0.org/wfwcomment.php?cid=7407

    1
    http://www.c0t0d0s0.org/rss.php?version=2.0&type=comments&cid=7407
    

    nospam@example.com (Joerg Moellenkamp)
    ibd driver. Let's assume you have to instances of the ibd driver running (ibd0 and ibd1). In this case you have to change one line at the end of /kernel/drv/ibd.conf file to enable_rc=1,1; and reload the ibd driver respectively reboot the system. After that you  ibd devices should show an mtu size of 65520 bytes instead of 2044.



PS: The process for Solaris 11 is better, as you just use dladm for it. However connected mode is the default there anyway. In Solaris 10 unreliable datagram was kept as the default, as one of the rules in Solaris is that you have to opt-in to such changes between updates. 
    ]]>

    Mon, 03 Oct 2011 23:03:36 +0200
    http://www.c0t0d0s0.org/archives/7407-guid.html
    http://creativecommons.org/licenses/by-sa/2.5/
http://www.c0t0d0s0.org/archives/7407-How-to-activate-IPoIB-Connected-mode-in-Solaris-10-Update-9.html

    
    http://feedproxy.google.com/~r/C0t0d0s0org-Solaris/~3/5wnZlf3kPSA/7402-Hunting-red-herrings.html
            English
            Solaris
    
    http://www.c0t0d0s0.org/archives/7402-Hunting-red-herrings.html#comments
    http://www.c0t0d0s0.org/wfwcomment.php?cid=7402

    16
    http://www.c0t0d0s0.org/rss.php?version=2.0&type=comments&cid=7402
    

    nospam@example.com (Joerg Moellenkamp)
    


This is the story of such a search. It started with a telephone call of a colleague. He got my name from another colleague. An Oracle database running on a Solaris system, the datafiles and logs are located on a Veritas File System. The customer saw massive delays (in the range of hundreds of seconds) when excuting certain commands. One of the commands  was “truncate table”.



A hypothesis - but the proof?
And in the beginning it started with a red herring. 

semtimedop(28, 0xFFFFFFFF7FFFD644, 1, 0xFFFFFFFF7FFFD630) Err#11 EAGAIN


In this case the thread is trying to execute something on  a semaphore, but it wasn’t able to do so. However the semtimedop is timebombed. When the timeout is reached without being able to execute on the semaphore , it terminates with error 11. All the timeouts were consistent with the waiting time seen from the SQL commands perspective.



Obviously the customer and other involved parties were tempted to see this as the problem,  but already thought that this may be just the harbinger of bad news. And after a short look into the truss files, I was pretty sure that they were right with their doubts in regard of passing the . It was just the harbinger of bad news.



After a short amount of research I suspected, that we were talking about a locking problem here. There was just a problem: vxfs. At first I worked seldomly with it, thus it’s not really my center of expertise. 



One point that diverted the attention of the customer from the locking stuff is a small but important difference: The customer knew that Oracle likes Direct I/O. With UFS the "Direct I/O" is doing a little bit more than just making the I/O direct by disabling buffering. It also removes the inode r/w lock mandated by POSIX rules.The customer knew about UFS Direct I/O that and thus activated Direct I/O on vxfs. And thus I found lines like /oracle/importantdatabase/oradata1 on /dev/vx/dsk/importantdatabase/oradata1 read/write/setuid/devices/mincache=direct/convosync=direct/delaylog/largefiles/ioerror=mwdisable/mntlock=VCS/dev=51836b0 on Thu Mar 17 20:14:11 2011



However i stil suspected a lock contention problem, and had a reason for it: Direct I/O isn't the same with vxfs than it's in UFS. In vxfs Direct I/O is really just the direct part. It doesn't enable concurrent I/O  (explain that moniker later) to a file. The removal of the inode r/w-lock isn't part of the feature. You have to use either Quick I/O (QIO) or the  ODM module for vxfs. As both features weren't activated, that was the moment where i told the customer "Hey, choose the ODM module for vxfs or QIO, activate it and the problem should go away". Both remove that lock contention and thus are of big help in order to get better Oracle performance when using vxfs. Just to remove a misunderstanding: ODM (Oracle Disk Management) is an API in Oracle, not of Veritas. Oracles DNFS (direct NFS) is implemented via an ODM module as well.



The problem: You used to pay for both vxfs, neither of them is really cheap and before doing the change, the customer wanted to know that i was right with my diagnosis (according to the release notes, ODM and QIO are now part of the SF except in basic). 



I wrote of two problems, but just wrote of one so far. Normally, finding out this inode rwlock contention problems are quite easy to find . But not in this case. vxfs is different than UFS in a multitude of ways. It doesn’t use the locking primitives of Solaris but has its own instead. And thus all values reported by prefered diagnosis tools were pretty useless. Damned … how should you find problems, when your instruments can’t show the problems.  Without instrumentation troubleshooting is just guesswork and experience.



At this point a question on a mail alias (it’s great to have people on internal aliases, that have forgotten more about Solaris than I know) and some research via google yielded the same result in a few minutes of time: vxfs`vx_rwsleep_rec_lock is the function waiting on/implementing the posix inode rw lock. Now I was back in the game and I was able to use all the nice things of the operating system i prefer.



Digging in the dirt
I asked the customer to put a dtrace script into a script that is executed in the moment of the wait:

bash-3.00# cat lock.d

tick-1s
/ i++ >= 60 /
{
exit(0);
}

fbt:vxfs:vx_rwsleep_rec_lock:entry
{
        self->in = timestamp
}

fbt:vxfs:vx_rwsleep_rec_lock:return
/self->in/
{
       locked[stack()] = quantize(timestamp - self->in);
        self->in = 0;
}


The result was interesting, as it clearly showed a peak of 307 events in the range 34359738368 nanoseconds (34.36 seconds) to 68719476735 nanoseconds (68.72 seconds). 

              vxfs`vx_write_common+0x2a8
              vxfs`vx_write+0x28
              genunix`fop_write+0x20
              genunix`pwrite+0x22c
              unix`syscall_trap+0xac

           value  ------------- Distribution ------------- count
             512 |                                         0
            1024 |                                         1
            2048 |@@@@@                                    160
            4096 |@@@@@@@@@@@@@@@@                         528
            8192 |@@@@                                     144
           16384 |@@                                       57
           32768 |                                         0
           65536 |                                         8
          131072 |                                         0
          262144 |                                         2
          524288 |                                         1
         1048576 |                                         4
         2097152 |                                         7
         4194304 |                                         8
         8388608 |                                         13
        16777216 |@                                        23
        33554432 |@                                        25
        67108864 |@                                        23
       134217728 |                                         1
       268435456 |                                         0
       536870912 |                                         0
      1073741824 |                                         0
      2147483648 |                                         0
      4294967296 |                                         0
      8589934592 |                                         0
     17179869184 |                                         0
     34359738368 |@@@@@@@@@                                307
     68719476736 |                                         0


This was especially interesting as the same dtrace script didn't showed such a peak during times where the system ran flawlessly.

              vxfs`vx_write_common+0x2a8
              vxfs`vx_write+0x28
              genunix`fop_write+0x20
              genunix`pwrite+0x22c
              unix`syscall_trap+0xac

           value  ------------- Distribution ------------- count
             512 |                                         0
            1024 |                                         3
            2048 |@@@@@@@@                                 77
            4096 |@@@@@@@@@@@@@                            122
            8192 |@@@@@@@@@                                89
           16384 |@@@@@@@@                                 77
           32768 |                                         0
           65536 |                                         0
          131072 |                                         0
          262144 |                                         0
          524288 |                                         0
         1048576 |@                                        9
         2097152 |                                         1
         4194304 |                                         1
         8388608 |                                         0


Okay ... well ... next step ... what parts of the system were executing this vxfs`vx_rwsleep_rec_lock function. I could have used dtrace for this task as well, but i wanted some additional insight in one step. Thus i used a nice little command of the modular debugger in Solaris: # echo "::threadlist -v" | mdb -k 

The output is quite long on a loaded solaris system. It prints something like this for each thread:

ffffff02d153ce40 ffffff02cefdb540 ffffff02cefff1a0   1  59 ffffff02d153d01e
  PC: _resume_from_idle+0xf1    CMD: /lib/svc/bin/svc.startd
  stack pointer for thread ffffff02d153ce40: ffffff000f91ccc0
  [ ffffff000f91ccc0 _resume_from_idle+0xf1() ]
    swtch+0x145()
    cv_wait_sig_swap_core+0x174()
    cv_wait_sig_swap+0x18()
    cv_waituntil_sig+0x13c()
    lwp_park+0x157()
    syslwp_park+0x31()
    sys_syscall32+0xff()
I hate multiple line outputs when searching for patterns. There is nothing better than two monitors, an terminal streched on both and the two glibberish grep-implementations  on the front side of your skull. But this works best, if one event is just in one line.



So i did some grepsed-fu on it.

echo "::threadlist -v" | mdb –k | sed 's:^$:§:' | tr -d '\n' | tr '§' '\n' > mdbout
.

Each thread is now in a single line. Yeah … perhaps there is a more elegant way to do this, but that was the first that came into my mind 

00000300646655e0      600a6796cb8      600d62e14d8   1  60      600e64f4520  PC: cv_wait+0x38    CMD: ora_dbwriter7_importantdatabase  stack pointer for thread 300646655e0: 2a1075f4da1  [ 000002a1075f4da1 cv_wait+0x38() ]    vx_rwsleep_rec_lock+0x70()    vx_write_common+0x2a8()    vx_write+0x28()    fop_write+0x20()    pwrite+0x22c()    syscall_trap+0xac()


Just a quick check. 

# cat mdbout | grep "vx_rwsleep" | wc -l
1008


At the moment of the hang, 1008 processes were in vxfs`vx_rwsleep_rec_lock. That was interesting. Even more interesting were the list of commands that had threads in the mentioned function. It's column 10 in the threadlist in it's concatenated form.

# grep "vx_rwsleep_rec_lock" mdbout  | tr  -s ' ' | cut -d " " -f 10 | sort | uniq
ora_dbwriter1_importantdatabase
ora_dbwriter3_importantdatabase
ora_dbwriter5_importantdatabase
ora_dbwriter7_importantdatabase


When you further dig down into the large heap of data:

# grep "ora_dbwriter3_importantdatabase" mdbout  | wc -l
258
# grep "ora_dbwriter3_importantdatabase" mdbout | grep -v "vx_rwsleep_rec_lock" | wc -l
7


From all this threads belonging to the ora_dbwriter3_importantdatabase just seven weren't in the vx_rwsleep_rec_lock function.



At that moment i thought: That isn't a smoking gun, that's a smoking howitzer.



An attempt to explain
Most threads excuting this function are part of the database writers. When you think about it, that's not so astonishing, especially when you think about the nature of an rwlock. At first: There is a rwlock for each inode in a filesystem. Their function: Multiple readers can get the lock and so they can read concurrently from the file, but just one writer is able to hold it and thus to write into the file. Equally important: You can't write to the file as long one or more readers is in the codepath protected by the rwlock for this file, and no one can read from the file as long there is a writer in protected codepath.

 

In really basic rwlock implementations this can lead to writer starvation, as it's hard for the writer to get the lock, because all readers have to relinquish the rwlock and no new readers should start before the writer can get the lock. Out of this reason, the Solaris threads implementation tends to favour writers before readers. However when you have many writers, it may take a long time before the backlog of writes.  Blindly prefering writers is not a solution as well, because then readers would starve which is even more problematic, because reads are always synchronous by nature. As i wrote at other locations. While a system can chose the time of an physical write to some extent, it can't chose the time of a read. A function won't execute as long the data isn't available. But that's out of scope of this article. For the capability to write and read in parallel to a file the name cocurrent I/O was coined.



I just wrote that it can take a moment before the backlog of writes has been executed. In this case it was even worse: The inode r/w lock adds insult to injury. Because basically the inode r/w lock limits you to just a single write I/O operation in parallel to a file, no matter how many HBA, how many disks you have in your system. And now you've made a while out of a moment. Even when the changes in the file are totally unrelated, e.g. changing a block belonging to the user table stored in it and another block in the article database or you want to read a block into the sga containing the customer database and writing the new salary for the promited assistant. You can't do this in parallel due to the inode rw lock. And with many updates in your workload it's not that astonishing that database writer threads start to twiddeling fingers in an increasing number in order to wait for their turn to write to the file.



You may ask yourself, why the heck there is such a mechanism. The r/w lock is something mandatory in order to be Posix compliant. You need it to ensure write ordering and consistent reads, when updates occur in parallel to read. Obviously you really want such a protection when working with files. However especially with databases a file is just a container for a large heap of things. Independent things. And things are now different.



Out of this reason there were some developments in the database realm to get rid of the inode rwlock and put this mechanism elsewere. Oracle allows you to use a raw disk, and so it has to do the consistent read and write ordering stuff anyways and as it’s aware of the inner structure of the heaps of data, it can do it with a much greater granularity than just per inode and thus per file. The inode r/w lock is just a bottleneck without any use in this case.



Out of this reason Direct I/O of UFS for example offers a mode that removes the lock. It's not the way, that those write ordering things or consistency protections are away. They are just in a layer that knows more about the structure inside the file and thus can do a better job at doing this job. vxfs knows similar mechanisms. QIO or ODM don't have such an inodewise locking. They are working differently compared with UFS direct I/O but as an earlier chancelor of the Federal Republic of Germany said: Outcome matters.



One question was still open. Why was this problem reproducible by a "TRUNCATE TABLE" command? That’s pretty easy however you have to dig deep into the internals of Oracle. When Oracle executes a TRUNCATE TABLE command, it checkpoints the database. In such a situation it writes all dirty blocks from the SGA into the database datafiles. This must be done for recovery purposes. 



Such checkpointing may  trigger a storm of writes via the database writer, especially when you have a SGA with a lot of dirty blocks. The checkpoint has to complete, before the TRUNCATE TABLE executes. And then we are at another red herring at the end: It's not the TRUNCATE TABLE command that was slow ... it's the checkpoint occuring before. You can check this pretty easy, when a "TRUNCATE TABLE" takes too long for your taste, trigger a checkpoint manually and do the TRUNCATE TABLE directly afterwards. TRUNCATE TABLE does still a checkpoint, but as you've already cleaned up the SGA from dirty buffers, it doesn't have to do much writing. It should run much faster now.



Conclusion
At the end i had to tell the customer, that in essence everything works as designed. It would be a bug, when the system would act just a little bit different differently. However that's seldom the answer a customer wants. 



So: The solution for the issue? It's as old as it's easy. Getting rid of the inode rwlock. Get concurrent I/O: Either by using raw disks, by using ASM, by using UFS or by using ODM or QIO for vxfs. I just can reiterate something i've already said: When you put your Oracle database file into a filesystem, you want to use direct I/O and concurrent I/O!

 
    ]]>

    Mon, 15 Aug 2011 21:13:58 +0200
    http://www.c0t0d0s0.org/archives/7402-guid.html
    http://creativecommons.org/licenses/by-sa/2.5/
http://www.c0t0d0s0.org/archives/7402-Hunting-red-herrings.html

    
    http://feedproxy.google.com/~r/C0t0d0s0org-Solaris/~3/dl2AU33IE_k/7394-Migrating-your-notebook-from-a-smaller-to-a-larger-disk.html
            English
            Solaris
            Sun/Oracle
    
    http://www.c0t0d0s0.org/archives/7394-Migrating-your-notebook-from-a-smaller-to-a-larger-disk.html#comments
    http://www.c0t0d0s0.org/wfwcomment.php?cid=7394

    1
    http://www.c0t0d0s0.org/rss.php?version=2.0&type=comments&cid=7394
    

    nospam@example.com (Joerg Moellenkamp)
    the "workaround" for ZFS resizing, however Christophe does takes this significantly forward and does this for boot disks.



OS: Solaris 11 express 2010.11 + SRU8 

Steps:

1) Copy data + OS on the new HDD
    a) connection of the new 500 GB HDD as an external USB HDD (using a USB external HDD box)
    b) creation of a Solaris 2 partition with fdisk and make it active (bootable)
         # fdisk /dev/rdsk/c4t0d0p0
    c) with the format command, create a partition s0 with all cylinders except cylinder 0
    d) Mirroring the existing ZFS pool (rpool) to the new HDD
         # zpool attach -f rpool c1t0d0s0 c4t0d0s0
        notes:
        - c1t0d0 is the 80 GB HDD (old HDD)
        - c4t0d0 is the 500 GB HDD (new HDD)
        - the option -f is necessary to bypass the warning "partition 0 overlaps partition 2"
    e) wait for the sync to be finished (with zpool status)
    f) Install Grub on the new HDD
        #  installgrub -m /boot/grub/stage1 /boot/grub/stage2 /dev/rdsk/c4t0d0s0
    g) Split the pool rpool by detaching to new HDD to create a new pool
        # zpool split rpool rpool2 c4t0d0s0
        note: I chose not to detach the old HDD because I wanted it to be usable in case of problem

2) Shutdown OS and laptop, disconnect the USB external HDD and replace the internal 80 GB HDD by the new one

3) Rename the new pool rpool2 to rpool
     - Boot on a Solaris 11 Express LiveCD or the network using AI
    note: In my case, I used an AI server I had installed before (Solaris 11 express 2010.11 with no SRU)
    - zpool import rpool2 rpool        to rename the pool
    - zpool export rpool                    to export it so that there is no warning in step 4

4) Boot on the new HDD
    - It works just fine, but the pool size is still the size of the old HDD (80 GB)
      altough it uses a 500 GB partition (c4t0d0s0)

5) Increase the pool size to use the whole partition
     # zpool set autoexpand=on rpool 


The autoexpand really does an large amount of the trick. The size of a mirrored pool is always the size of the smallest disk. When you have an 80 GB and a 500 GB disk, the size of the pool is 80 GB. Remove the 80 GB disk. The smallest disk is now 500 GB and the size of the pool is 500 GB now as well, as long as you've activated autoexpand. 
    ]]>

    Thu, 21 Jul 2011 14:05:11 +0200
    http://www.c0t0d0s0.org/archives/7394-guid.html
    http://creativecommons.org/licenses/by-sa/2.5/
http://www.c0t0d0s0.org/archives/7394-Migrating-your-notebook-from-a-smaller-to-a-larger-disk.html

    
    http://feedproxy.google.com/~r/C0t0d0s0org-Solaris/~3/q-vjLKb_wrg/7370-A-little-change-of-queues.html
            English
            Solaris
            Sun/Oracle
    
    http://www.c0t0d0s0.org/archives/7370-A-little-change-of-queues.html#comments
    http://www.c0t0d0s0.org/wfwcomment.php?cid=7370

    2
    http://www.c0t0d0s0.org/rss.php?version=2.0&type=comments&cid=7370
    

    nospam@example.com (Joerg Moellenkamp)
    


There is a slight change in the default parameters of ZFS in Update 9. It’s related to the parameter zfs:zfs_vdev_max_pending  . This parameter controls, how many I/O requests can be pending per vdev.  For example when you have 100 disks visible from your OS with a zfs:zfs_vdev_max_pending of 2, you have 200 request outstanding at maximum. When you have 100 disks hidden behind your storage controller just showing a single LUN, you will have – you will know it – 2 pending requests at maximum.



You may think, that you could increase the queue depth without end, but as usual this is a tradeoff game and not that easy, longer queue depths may increase latency of the commands. Experience showed that certain queue depth delivered the best performance on most installations.



However the installed landscape changes and sometimes you have to adjust things. Exactly this happened a while ago in Opensolaris. And it seems that this change moved into Solaris. The default for zfs:zfs_vdev_max_pending is 10 at the moment. You can check this:

# echo zfs_vdev_max_pending::print | mdb –kw
0xa
#
0xa in decimal is 10.



And this is a wise choice for most implementations out there. But it was different on older versions. I checked it on U7, i asked my twitter/facebook contacts to make quick check on U8 as i was to lazy to install it: 

# echo zfs_vdev_max_pending::print | mdb –kw
0x23
#


0x23 in decimal is 35 and 35 was the default up to Update 8 of Solaris 10.



So essentially  the queues are less deep than before. For JBODs this is most often a good thing, as each vdev and thus each LUN has its own queue of 10 pending I/Os. For a single LUN hiding many disks sometimes not. So how do you change it back to the old value?

You can change it dynamically:

# echo zfs_vdev_max_pending/W0t35 | mdb –kw


To make this change boot-persistent you have to add a line to /etc/system:

set zfs:zfs_vdev_max_pending = 35 


Sometimes even an higher value may be indicated with very large numbers of disks behind your controller forming a single LUN.

How do you know if this decreased queue depth is a problem for you at all? The command iostat will help you:

jmoekamp@hivemind:~$ iostat -xdn
                    extended device statistics
    r/s    w/s   kr/s   kw/s wait actv wsvc_t asvc_t  %w  %b device
    6,3    1,9  525,9   31,2  0,1  0,0   16,4    6,0   2   3 c3d0
   17,1    1,0 1676,0    8,0  0,2  0,1   11,4    4,8   4   4 c3d1
    6,4    1,9  525,8   31,2  0,1  0,0   14,1    4,8   2   2 c4d0
   17,1    1,0 1675,9    8,0  0,2  0,1   12,9    4,7   4   4 c4d1
    0,0    0,0    0,0    0,0  0,0  0,0    0,0    0,0   0   0 gsdbc
jmoekamp@hivemind:~$


If you see the column  actv at or near the number of zfs:zfs_vdev_max_pending, it’s worth a try. Otherwise not. 

 
    ]]>

    Wed, 06 Jul 2011 11:50:58 +0200
    http://www.c0t0d0s0.org/archives/7370-guid.html
    http://creativecommons.org/licenses/by-sa/2.5/
http://www.c0t0d0s0.org/archives/7370-A-little-change-of-queues.html

    
    http://feedproxy.google.com/~r/C0t0d0s0org-Solaris/~3/UYANBjqdUEU/7366-New-Hivemind-Powermanagement.html
            English
            Solaris
    
    http://www.c0t0d0s0.org/archives/7366-New-Hivemind-Powermanagement.html#comments
    http://www.c0t0d0s0.org/wfwcomment.php?cid=7366

    0
    http://www.c0t0d0s0.org/rss.php?version=2.0&type=comments&cid=7366
    

    nospam@example.com (Joerg Moellenkamp)
    HP N36L microserver. The answer is yes.



Best way to check this is via kstat. If kstat shows multiple frequencies as supported frequencies, it supports Power Management for the processor:

jmoekamp@hivemind:~$ kstat -m cpu_info | grep "supported_frequencies_Hz"
        supported_frequencies_Hz        800000000:1100000000:1300000000
        supported_frequencies_Hz        800000000:1100000000:1300000000
However before Solaris really uses it, you have to configure powermanagement. At first add or change the following lines in /etc/power.conf:

cpupm            enable
cpu_deep_idle enable
cpu-threshold    10s
Afterwards run the command pmconfig once. Now keep the system idling for 10 seconds and check the frequencies the system runs at:
jmoekamp@hivemind:~# kstat -m cpu_info | grep "current_clock_Hz"  
        current_clock_Hz                800000000
        current_clock_Hz                800000000


 
    ]]>

    Sun, 03 Jul 2011 07:33:27 +0200
    http://www.c0t0d0s0.org/archives/7366-guid.html
    http://creativecommons.org/licenses/by-sa/2.5/
http://www.c0t0d0s0.org/archives/7366-New-Hivemind-Powermanagement.html

    
    http://feedproxy.google.com/~r/C0t0d0s0org-Solaris/~3/ic_IE-WyMes/7348-Result-of-the-How-long-do-you-wait-before-Solaris-11-gets-on-your-prod-systems.html
            English
            Solaris
            Sun/Oracle
    
    http://www.c0t0d0s0.org/archives/7348-Result-of-the-How-long-do-you-wait-before-Solaris-11-gets-on-your-prod-systems.html#comments
    http://www.c0t0d0s0.org/wfwcomment.php?cid=7348

    16
    http://www.c0t0d0s0.org/rss.php?version=2.0&type=comments&cid=7348
    

    nospam@example.com (Joerg Moellenkamp)
    
 
    ]]>

    Thu, 23 Jun 2011 16:20:48 +0200
    http://www.c0t0d0s0.org/archives/7348-guid.html
    http://creativecommons.org/licenses/by-sa/2.5/
http://www.c0t0d0s0.org/archives/7348-Result-of-the-How-long-do-you-wait-before-Solaris-11-gets-on-your-prod-systems.html

    
    http://feedproxy.google.com/~r/C0t0d0s0org-Solaris/~3/N_uC1cWDivM/7324-Vortrag-Deduplication-auf-der-Froson.html
            English
            Solaris
            The IT Business
    
    http://www.c0t0d0s0.org/archives/7324-Vortrag-Deduplication-auf-der-Froson.html#comments
    http://www.c0t0d0s0.org/wfwcomment.php?cid=7324

    0
    http://www.c0t0d0s0.org/rss.php?version=2.0&type=comments&cid=7324
    

    nospam@example.com (Joerg Moellenkamp)
    Froscon 2011 angenommen worden ist. Die  Froscon findet dieses Jahr am 20./21. August 2011 an Hochschule Bonn-Rhein-Sieg statt. Ich werde dort eine vollständig überarbeitete Version meines Deduplication-Vortrags von der GUUG halten. An vielen stellen allgemeiner und nicht so ZFS-zentrisch, an anderen Stellen allerdings sehr viel ZFS-zentrischer, da ich anhand des Sourcecodes von ZFS einige Konzepte erklaeren will. 
    ]]>

    Mon, 06 Jun 2011 09:57:41 +0200
    http://www.c0t0d0s0.org/archives/7324-guid.html
    http://creativecommons.org/licenses/by-sa/2.5/
http://www.c0t0d0s0.org/archives/7324-Vortrag-Deduplication-auf-der-Froson.html

    
    http://feedproxy.google.com/~r/C0t0d0s0org-Solaris/~3/vH10Vchglhk/7314-Solaris-1011x-hardware-compatibility-list.html
            English
            Solaris
            Sun/Oracle
    
    http://www.c0t0d0s0.org/archives/7314-Solaris-1011x-hardware-compatibility-list.html#comments
    http://www.c0t0d0s0.org/wfwcomment.php?cid=7314

    0
    http://www.c0t0d0s0.org/rss.php?version=2.0&type=comments&cid=7314
    

    nospam@example.com (Joerg Moellenkamp)
    http://www.oracle.com/webfolder/technetwork/hcl/index.html 
    ]]>

    Thu, 02 Jun 2011 21:40:16 +0200
    http://www.c0t0d0s0.org/archives/7314-guid.html
    http://creativecommons.org/licenses/by-sa/2.5/
http://www.c0t0d0s0.org/archives/7314-Solaris-1011x-hardware-compatibility-list.html

    
    http://feedproxy.google.com/~r/C0t0d0s0org-Solaris/~3/bs0TzWZ060Y/7296-SNIA-releases-testing-spec-for-SSD.html
            English
            Solaris
    
    http://www.c0t0d0s0.org/archives/7296-SNIA-releases-testing-spec-for-SSD.html#comments
    http://www.c0t0d0s0.org/wfwcomment.php?cid=7296

    0
    http://www.c0t0d0s0.org/rss.php?version=2.0&type=comments&cid=7296
    

    nospam@example.com (Joerg Moellenkamp)
    release of a testing specification for SSD performance is really welcome to me, as you often compare apples with pears and pears with potatoes. Looks like the SSD market gets more mature. 
    ]]>

    Tue, 24 May 2011 08:09:33 +0200
    http://www.c0t0d0s0.org/archives/7296-guid.html
    http://creativecommons.org/licenses/by-sa/2.5/
http://www.c0t0d0s0.org/archives/7296-SNIA-releases-testing-spec-for-SSD.html

    
    http://feedproxy.google.com/~r/C0t0d0s0org-Solaris/~3/uD3JlQisz1o/7289-Darren-Moffat-about-encrypted-swap-and-vartmp.html
            English
            Solaris
            Sun/Oracle
    
    http://www.c0t0d0s0.org/archives/7289-Darren-Moffat-about-encrypted-swap-and-vartmp.html#comments
    http://www.c0t0d0s0.org/wfwcomment.php?cid=7289

    0
    http://www.c0t0d0s0.org/rss.php?version=2.0&type=comments&cid=7289
    

    nospam@example.com (Joerg Moellenkamp)
    how to encrypt swap and /var/tmp in Solaris 11 Express. 
    ]]>

    Sun, 22 May 2011 08:42:01 +0200
    http://www.c0t0d0s0.org/archives/7289-guid.html
    http://creativecommons.org/licenses/by-sa/2.5/
http://www.c0t0d0s0.org/archives/7289-Darren-Moffat-about-encrypted-swap-and-vartmp.html

    
    http://feedproxy.google.com/~r/C0t0d0s0org-Solaris/~3/BKAZmPLVbGo/7271-ZFS-Dedup-Internals.html
            English
            Solaris
    
    http://www.c0t0d0s0.org/archives/7271-ZFS-Dedup-Internals.html#comments
    http://www.c0t0d0s0.org/wfwcomment.php?cid=7271

    3
    http://www.c0t0d0s0.org/rss.php?version=2.0&type=comments&cid=7271
    

    nospam@example.com (Joerg Moellenkamp)
    



 
Deduplication in ZFS is in-band. The deduplication occurs when you write to the disk.
 
The instrument to do deduplication is to use checksums. If a block has the same checksum as a block already written to the pool, it's considered as a duplicate and thus just a pointer to the already stored block is written to disk. To say it simplified)
 
As it's not reasonable to scan through all blocks on disk to read the checksums in order find a duplicate, the checksums are stored in a structure called deduplication table or DDT.
 
The DDTs are a so called AVL tree (that's obvious as there are lot of calls to the system wide avl tree code). This helps searching, so it doesn't just grep through the DDT. It's more intelligent.
 
The DDTs are managed as a ZAP object on-disk
 
ZAP is short for the ZFS Attribute processor
 
A ZAP object stores key value pairs used in many tasks organizing data stored in ZFS
 
ZAP data is metadata
 
Metadata is cached by the ARC
 
As it's cached in ARC, the ZAP blocks and the the data inside those blocks is object to the cache eviction policies of ARC
 
Just increasing memory size in order to get more room for the DDT  may be insufficient, as the space metadata can occupy in ARC is limited.
 
Per default it's limited to 25% of the maximum ARC size (see http://src.opensolaris.org/source/xref/onnv/onnv-gate/usr/src/uts/common/fs/zfs/arc.c#3478">line 3478 of arc.c)
 
The maximum ARC size is 3/4 of the systems memory or all-1GB, whatever is higher.
 
Both values are user-tunable.
 
You can set the amount of memory for the metadata to a user-chosen value with zfs:zfs_arc_meta_limit= in /etc/system.
 
That may be necessary when you know that you do a lot of deduplication tasks and the performance increase by keeping all or most of the DDT in memory outweights the loss of memory for caching data. For example mass importing data from a non-deduplicated storage may such a task.
 
 
The size of the ddt may vary over versions, at snv133 the size of object is
 
jmoekamp@hivemind:~# echo ::sizeof ddt_entry_t | mdb -k 
 sizeof (ddt_entry_t) = 0x178
Or decimal - 376 Bytes.
Such an entry exists for each unique block written to disk while deduplication is activated
Thus blocks written while deduplication is deactivated are not getting deduplicates (obviously), however you can't deduplicate other locks with them as well
So obviously a pool just using 128k blocks may need less memory to store the ddt than a pool just using 512 byte blocks
Everything stored in ARC may be moved to the L2ARC. So an SSD may be used to increase the space to cache the DDT. Due to the nature of SSD (no head positioning latency) is a much better place to search for the nescessary DDT parts than a rotating head.
However keep in mind that L2ARC doesn't come for free memorywise. Everything stored in L2ARC still leaves some information in the ARC. The relevant metadata structure is arc_buf_hdr_t (see line 431 of arc.c:
 
jmoekamp@hivemind:~# echo ::sizeof arc_buf_hdr_t | mdb -k
sizeof (arc_buf_hdr_t) = 0xa8
Or decimal - 178 bytes.
Just before you are asking about substituting a 376 Byte object by an 178 byte object - the ARC doesn't store a single dde entry in a ZAP block, they are storing multiple ones in it.
  
 
The deduplication is not done when writing to the ZIL, it's done when writing it finally into the pool. Thus the deduplication is not done when the OS blocks for the write when doing a sync write, instead it's part of an async process afterwards. However it's possible that you will see write throttling kicking in, when the system isn't capable to to write all the dirty pages away timey.
 
Access pattern for the blocks containing DDT data not in memory is very random.
 
System with large pools with small memory areas will not work well, as each write will result in a larger number of reads in order to get the data from the DDT.
 
The more unique blocks you have in the DDT, the less fast dedup will work, given a small amount of memory. A 1 gig memory in a  the box and several TB of storage with very small recordsizes will be a sure way to yield horrible performance.
 
Performance will be best, when the complete DTT fits in memory.
 
Systems with medium memory size and SSD will yield good results
 
However everything depends on the number of blocks  writen after activation of the deduplication, as the size of the DDT.
 
In short: Small pool, lot of memory - yeah, really big pool, minimal memory - nah.Truth is somewhat between that...
 
    ]]>

    Mon, 02 May 2011 12:15:37 +0200
    http://www.c0t0d0s0.org/archives/7271-guid.html
    http://creativecommons.org/licenses/by-sa/2.5/
http://www.c0t0d0s0.org/archives/7271-ZFS-Dedup-Internals.html

    
    http://feedproxy.google.com/~r/C0t0d0s0org-Solaris/~3/l4ZnQXrrYLM/7266-Recommended-DTrace-by-Brendan-Gregg-and-Jim-Mauro.html
            English
            Solaris
    
    http://www.c0t0d0s0.org/archives/7266-Recommended-DTrace-by-Brendan-Gregg-and-Jim-Mauro.html#comments
    http://www.c0t0d0s0.org/wfwcomment.php?cid=7266

    7
    http://www.c0t0d0s0.org/rss.php?version=2.0&type=comments&cid=7266
    

    nospam@example.com (Joerg Moellenkamp)
    


It was the book "DTrace" written by Brendan Gregg (the guy shouting at disk arrays) and Jim Mauro. In one short sentence:It's great ... it's really great. I recommend it for every Solaris admin. When you think that DTrace is a rather esoteric topic for developers, read that book. If you still think this after reading the book, take out the comic you placed in the middle before  Fun aside: This book explains the very basics of DTraces as well as a vast number of examples how to distill important information for debugging and performance analysis out of the system with DTrace. To do so, the authors look into each main subsystem (memory, network or disk i/o), doing some basic diagnosis with a DTrace oneliner just to show the depth of the rabbit hole with some more sophisticated scripts later. 



Probably you fill find many examples somewhere in the Internet, however this book is a great reference for this examples when you need them most (... when you manager stands right behind you asking you questions what happens at the moment).



I i have stated before: It's a recommended read! ! 
    ]]>

    Tue, 26 Apr 2011 19:37:54 +0200
    http://www.c0t0d0s0.org/archives/7266-guid.html
    http://creativecommons.org/licenses/by-sa/2.5/
http://www.c0t0d0s0.org/archives/7266-Recommended-DTrace-by-Brendan-Gregg-and-Jim-Mauro.html

    
    http://feedproxy.google.com/~r/C0t0d0s0org-Solaris/~3/mQtGQ7YwZYw/7262-Parasitic-Hadoop-for-backups.html
            English
            Solaris
            Toolbox
    
    http://www.c0t0d0s0.org/archives/7262-Parasitic-Hadoop-for-backups.html#comments
    http://www.c0t0d0s0.org/wfwcomment.php?cid=7262

    0
    http://www.c0t0d0s0.org/rss.php?version=2.0&type=comments&cid=7262
    

    nospam@example.com (Joerg Moellenkamp)
    parasitic hadoop in order to use unused resources on systems not loaded. An hadoop node installed on machines used for different things and configured in a way, that it just uses the resources left over by the host (the application you've bought the server for). iIn my experience, the most unloaded system part are the boot disks (to big and often not loaded). Ben Rockwood of cuddletech.com had a great idea: Using the Hadoop Distributed File System to store ZFS dumps. Read more about it in his blog. 
    ]]>

    Tue, 19 Apr 2011 06:11:16 +0200
    http://www.c0t0d0s0.org/archives/7262-guid.html
    http://creativecommons.org/licenses/by-sa/2.5/
http://www.c0t0d0s0.org/archives/7262-Parasitic-Hadoop-for-backups.html

    
    http://feedproxy.google.com/~r/C0t0d0s0org-Solaris/~3/yJ46ng9Jjhs/7240-Stefan-Schneider-about-the-Solaris-11-Compatibility-Checker.html
            English
            Solaris
            Sun/Oracle
    
    http://www.c0t0d0s0.org/archives/7240-Stefan-Schneider-about-the-Solaris-11-Compatibility-Checker.html#comments
    http://www.c0t0d0s0.org/wfwcomment.php?cid=7240

    0
    http://www.c0t0d0s0.org/rss.php?version=2.0&type=comments&cid=7240
    

    nospam@example.com (Joerg Moellenkamp)
    compatibility tester is really for:

Being the project owner of the tool: Oracle takes extreme care that previous investments of software partners in Solaris 10 will be protected. The tool is spotting fairly extreme corner cases of usage of private or deprecated interfaces and commands. The tool tracks the delta of Solaris 10 (318000 files) to Solaris 11 Express at a command and symbol level. This is about common change management with a strong focus on frame work version creep and end of life of components which are outdated for more than a decade. A few examples and their mitigation are getting discussed here: http://www.scalingbits.com/solaris/compatibility
 
    ]]>

    Wed, 23 Mar 2011 17:57:20 +0100
    http://www.c0t0d0s0.org/archives/7240-guid.html
    http://creativecommons.org/licenses/by-sa/2.5/
http://www.c0t0d0s0.org/archives/7240-Stefan-Schneider-about-the-Solaris-11-Compatibility-Checker.html

    
    http://feedproxy.google.com/~r/C0t0d0s0org-Solaris/~3/L9VYG717v-o/7225-Image-Packaging-System-in-the-Spotlight-of-the-OTN.html
            English
            Solaris
            Sun/Oracle
    
    http://www.c0t0d0s0.org/archives/7225-Image-Packaging-System-in-the-Spotlight-of-the-OTN.html#comments
    http://www.c0t0d0s0.org/wfwcomment.php?cid=7225

    0
    http://www.c0t0d0s0.org/rss.php?version=2.0&type=comments&cid=7225
    

    nospam@example.com (Joerg Moellenkamp)
    "Oracle Solaris 11 Express Package Management with Image Packaging System (IPS)" 
    ]]>

    Thu, 17 Mar 2011 21:15:59 +0100
    http://www.c0t0d0s0.org/archives/7225-guid.html
    http://creativecommons.org/licenses/by-sa/2.5/
http://www.c0t0d0s0.org/archives/7225-Image-Packaging-System-in-the-Spotlight-of-the-OTN.html

    
    http://feedproxy.google.com/~r/C0t0d0s0org-Solaris/~3/6tEmP4NJtJ0/7203-Cloud-analytics.html
            English
            Solaris
    
    http://www.c0t0d0s0.org/archives/7203-Cloud-analytics.html#comments
    http://www.c0t0d0s0.org/wfwcomment.php?cid=7203

    0
    http://www.c0t0d0s0.org/rss.php?version=2.0&type=comments&cid=7203
    

    nospam@example.com (Joerg Moellenkamp)
    analytics of cloud application, which in essence boils down to Solaris DTrace. That tools presented in the presentation somewhat remembers me more than a little of the Analytics tool of the S7000  
    ]]>

    Wed, 02 Mar 2011 08:31:56 +0100
    http://www.c0t0d0s0.org/archives/7203-guid.html
    http://creativecommons.org/licenses/by-sa/2.5/
http://www.c0t0d0s0.org/archives/7203-Cloud-analytics.html

    
    http://feedproxy.google.com/~r/C0t0d0s0org-Solaris/~3/uWZNT4KFZT0/7181-DTrace-book.html
            English
            Solaris
    
    http://www.c0t0d0s0.org/archives/7181-DTrace-book.html#comments
    http://www.c0t0d0s0.org/wfwcomment.php?cid=7181

    0
    http://www.c0t0d0s0.org/rss.php?version=2.0&type=comments&cid=7181
    

    nospam@example.com (Joerg Moellenkamp)
    DTrace book sample chapter: File Systems he offers a the table of content and the chapter about filesystems as a teaser. 
    ]]>

    Thu, 24 Feb 2011 09:43:33 +0100
    http://www.c0t0d0s0.org/archives/7181-guid.html
    http://creativecommons.org/licenses/by-sa/2.5/
http://www.c0t0d0s0.org/archives/7181-DTrace-book.html

    
    http://feedproxy.google.com/~r/C0t0d0s0org-Solaris/~3/fGPTehVkmd0/7171-Backporting.html
            English
            Solaris
            Sun/Oracle
    
    http://www.c0t0d0s0.org/archives/7171-Backporting.html#comments
    http://www.c0t0d0s0.org/wfwcomment.php?cid=7171

    2
    http://www.c0t0d0s0.org/rss.php?version=2.0&type=comments&cid=7171
    

    nospam@example.com (Joerg Moellenkamp)
    article written by Alan Hargreaves -as well Principal Field Technologist- to explain, why the version number of the Apache delivered with Solaris 10 doesn't automatically indicates that it's vulnerable against the attacks reported against Apache since that version:
Now, that being said you may also note after installation that it still identifies as Apache 2.0.63 and you may have concerns about vulnerabilities addressed in 2.0.64 mentioned on the Apache web site.



The way that we maintain Apache on Solaris 10 is not to drop in new releases as they happen, rather we take the fixes mentioned and backport them to our 2.0.63 codebase.
Alan, i hope linking to this article reliefs you from some additional calls about this topic  
    ]]>

    Mon, 21 Feb 2011 22:10:49 +0100
    http://www.c0t0d0s0.org/archives/7171-guid.html
    http://creativecommons.org/licenses/by-sa/2.5/
http://www.c0t0d0s0.org/archives/7171-Backporting.html

    
    http://feedproxy.google.com/~r/C0t0d0s0org-Solaris/~3/QlAdqW71CfA/7162-pstop-and-prun.html
            English
            Solaris
    
    http://www.c0t0d0s0.org/archives/7162-pstop-and-prun.html#comments
    http://www.c0t0d0s0.org/wfwcomment.php?cid=7162

    5
    http://www.c0t0d0s0.org/rss.php?version=2.0&type=comments&cid=7162
    

    nospam@example.com (Joerg Moellenkamp)
    


The test case
I wrote a small script to demonstrate both  commands.
#!/usr/bin/perl
while () {
 $zeit=time();
 print "... $zeit \n";
 sleep(2);
}
It just prints the current time every 2 seconds.
jmoekamp@hivemind:~$ ./stopstart.pl
... 1297973807
... 1297973809
... 1297973811
... 1297973813
... 1297973815
... 1297973817
... 1297973819
... 1297973821
... 1297973823
... 1297973825
... 1297973827
... 1297973858
... 1297973860
... 1297973862
... 1297973864
... 1297973866
... 1297973868
... 1297973870
^C
jmoekamp@hivemind:~$
You may have recognized that there is a 31 second gap between 1297973827 and 1297973858 for script that should print out the timestamps continiously every two seconds.



What i did in a second shell?
What happened here? At 1297973827 i ran the following command:
jmoekamp@hivemind:~# pstop 10103
jmoekamp@hivemind:~#
When you look at the process flags, you will see that the process has been stopped:
jmoekamp@hivemind:~# pflags 10103
10103:  /usr/bin/perl ./stopstart.pl
        data model = _ILP32  flags = MSACCT|MSFORK
 /1:    flags = STOPPED|ISTOP|ASLEEP  nanosleep(0x8047a00,0x8047a08)
        why = PR_REQUESTED
A moment before that i found out the process id via 
jmoekamp@hivemind:~# ps -ef | grep "stopstart.pl" | grep -v "grep"
jmoekamp 10103  9998   0 21:16:48 pts/2       0:00 /usr/bin/perl ./stopstart.pl
Then i just waited for half a minute and resumed execution with 

jmoekamp@hivemind:~# prun 10103
jmoekamp@hivemind:~#
So the perl scripted resumed to print the timestamps at 1297973858. When you check the pflags again you will see, that the the STOPPED and ISTOP flags went away:
jmoekamp@hivemind:~# pflags 10103
10103:  /usr/bin/perl ./stopstart.pl
        data model = _ILP32  flags = MSACCT|MSFORK
 /1:    flags = ASLEEP  nanosleep(0x8047a00,0x8047a08)

jmoekamp@hivemind:~#
 
    ]]>

    Thu, 17 Feb 2011 21:52:32 +0100
    http://www.c0t0d0s0.org/archives/7162-guid.html
    http://creativecommons.org/licenses/by-sa/2.5/
http://www.c0t0d0s0.org/archives/7162-pstop-and-prun.html

    
    http://feedproxy.google.com/~r/C0t0d0s0org-Solaris/~3/v_9VJEWvQy8/7149-Secure-Deployment-of-LDOMs.html
            English
            Solaris
            Sun/Oracle
    
    http://www.c0t0d0s0.org/archives/7149-Secure-Deployment-of-LDOMs.html#comments
    http://www.c0t0d0s0.org/wfwcomment.php?cid=7149

    0
    http://www.c0t0d0s0.org/rss.php?version=2.0&type=comments&cid=7149
    

    nospam@example.com (Joerg Moellenkamp)
    secure deployment of LDOMs. 
    ]]>

    Thu, 03 Feb 2011 19:09:34 +0100
    http://www.c0t0d0s0.org/archives/7149-guid.html
    http://creativecommons.org/licenses/by-sa/2.5/
http://www.c0t0d0s0.org/archives/7149-Secure-Deployment-of-LDOMs.html

    
    http://feedproxy.google.com/~r/C0t0d0s0org-Solaris/~3/snM0BJ8evv4/7147-Synchronicity-in-asynchronicity.html
            English
            Solaris
            Technology
    
    http://www.c0t0d0s0.org/archives/7147-Synchronicity-in-asynchronicity.html#comments
    http://www.c0t0d0s0.org/wfwcomment.php?cid=7147

    5
    http://www.c0t0d0s0.org/rss.php?version=2.0&type=comments&cid=7147
    

    nospam@example.com (Joerg Moellenkamp)
    


As you know there are two important concepts of executing writes. Synchronous and asynchronous or to say it differently blocking or non-blocking. When you trigger an asynchronous write, the write call comes back, right after you issued the call. An synchronous write call (by definition) is only allowed to come back, when the system has assured (as far it's technically possible) that the data is on some kind of nonvolatile storage.



Synchronous writes are absolutely essential for example for mail servers: The MTA can't send OK to the MUA or to another MTA the "OK, got the mail", as long it's sure the mail is nonvolatile on the other server, because the sending MTA deletes it from its queue after such a OK. A mail could be lost, when the power fails after the OK, but before the non-volatile storing of the mail. So you send the OK afterwards. How to you ensure, that the OK is after the nonvolatile storing? Yes ... it's done by a sync write.



It's the same for databases: One of the important foundations of ACID is the synchronous write. Without synchronous writes, forget about the D ...



However a synchronous write has a problem and it is because of its greatest advantage. It doesn't come back until the write has been completed. The problem is that you can't do anything else in this time in this thread.



Translating this to an example in the real world, you can imagine this like being the dispatcher of large group of employees. When you give your employees their tasks synchronously, you give your employees the job and wait for the completion before you dispatch another job. You can be sure, that the task has been completed before you throw the the piece of paper with the todo, however it's not really efficient.



Switching back to asynchronous writes isn't an option as well, as the only way to check, if the write call was really processed through the complete code path and the code paths inside the HBA and the controller adjacent to the disk, is to read the location (okay, they are other reasons to do so, however that a completely different discussion). The asynchronous write call gives you no feedback of that the write call has been completed in terms of writing stuff to nonvolatile storage. Basically you trust the operating system that it will do everything right and aside of this there is absolutely no guarantee that asynchronously written data is on disk after a system crash.



Translated to the real world example, this would be like giving a task to  an employee, throwing away the piece of paper where you wrote down the task ... and forget about it. Obviously you can work very fast this way, and often this works well, however when an employee quits or is absent due to illness, the job may not completed.



By the way: It's the same for reads. A reads blocks the calling process until the data has been delivered, and this time there is no way around with it with normal means. Reading is synchronous by necessity. The process can't say "Well ... when i have to wait for it ... i will take some other data available"



So, waiting processes until the write comes call returns, missing guarantees and no feedback on the other side.  And reads synchronous anyway. And even more devastating: When a write or read call blocks a thread, because it waits until all the stuff between the OS and the disk has been done, no matter how many parallel disks or filesystem you have. How do you get out of this challenge?



There is a way around that and this is Asynchronous I/O. The read and writes are still synchronous or asynchronous in the sense of blocking or non-blocking. So you can have basically an synchronous write in a asynchronous I/O model. This is important: As a colleague said it years ago: You can configure Oracle to use a asynchronous I/O model, but you can't tell Oracle to use non-synchronous write semantics. Remember the D.



To use this model, there are a number of new calls in the OS that support this asynchronous I/O modell.



The write call aiowrite or read call call aioread returns right after you issue the call. Sounds like asynchronous write semantics. However there is an important difference: At first the read or write is now run in a concurrent thread and the aioread or aiowrite returns immediately after they are issued. But contrary to the asynchronous write semantic, you get a feedback about the result of the write. 



There are essentially two important ways to yield this information, at first you could call the aiowait function. This returns as soon as a outstanding read or write triggered by aiowrite or aioread returns. It waits either forever (until an outstanding aio request completes), not at all ("Is something completed available ... no ... okay ... let's go further". You use this mode of operating to implement a polling mechanism) or wait for a certain time.



The other way is to implement a signal handler. Whenever a I/O request made by those calls, they will send the signal SIGPOLL, the signal handler can then dequeue the notification of a completed I/O request with aiowait (in fact, they have to because it's the only way  to get those notifications out of the queue)



However: Both mechanisms notify (SIGPOLL fires, aiowait returns), when one of outstanding asynchronous requests is completed. They doesn't report the I/O call that has been completed. You have to find which one. Most often this is done by scanning the return code buffers of the aiowrite/aioread requests after you have initialized them with a value expressing "in progress". When a i/o request has been completed, it's set to  something else, so you just have to scan for return code buffers, that are not on the state "in progress).



Translated into the real world, it's like dispatching a steady stream of tasks to your employees. However you don't monitor their progress yourself, but you have some colleagues for that. Those colleagues just send you a notification "Task xyz has been completed with that result" and could throw the piece of paper with the task away, as soon as this notification was send to you. You could do it at a jour-fixe, you could wait in front of your colleagues or you could wait for 5 minutes for completion notifications and then do something else for the next hours. 



The advantage of asynchronous I/O is obvious: The application can issue I/O requests without waiting for the completion of others, you can even issue asynchronous reads. By doing so a single application thread can have multiple I/O requests in flight. However you  addition you get still a feedback, that the I/O request has been successfully completed and you can use it in areas where you would have opted to use synchronous aka blocking write semantics in the past.  Important to know: Your application must be enabled by the developers to use an asynchronous I/O pattern.



 
    ]]>

    Thu, 03 Feb 2011 08:21:00 +0100
    http://www.c0t0d0s0.org/archives/7147-guid.html
    http://creativecommons.org/licenses/by-sa/2.5/
http://www.c0t0d0s0.org/archives/7147-Synchronicity-in-asynchronicity.html

    
    http://feedproxy.google.com/~r/C0t0d0s0org-Solaris/~3/yFObLH_xp84/7145-30-Day-Song-Challenge-Day-07-A-song-that-reminds-you-of-a-certain-event.html
            English
            Solaris
    
    http://www.c0t0d0s0.org/archives/7145-30-Day-Song-Challenge-Day-07-A-song-that-reminds-you-of-a-certain-event.html#comments
    http://www.c0t0d0s0.org/wfwcomment.php?cid=7145

    4
    http://www.c0t0d0s0.org/rss.php?version=2.0&type=comments&cid=7145
    

    nospam@example.com (Joerg Moellenkamp)
     
    ]]>

    Wed, 19 Jan 2011 21:41:52 +0100
    http://www.c0t0d0s0.org/archives/7145-guid.html
    http://creativecommons.org/licenses/by-sa/2.5/
http://www.c0t0d0s0.org/archives/7145-30-Day-Song-Challenge-Day-07-A-song-that-reminds-you-of-a-certain-event.html

    
    http://feedproxy.google.com/~r/C0t0d0s0org-Solaris/~3/bJIQyiLRHZM/7144-Would-be-a-nice-tagline-for-Solaris-.....html
            English
            Solaris
    
    http://www.c0t0d0s0.org/archives/7144-Would-be-a-nice-tagline-for-Solaris-.....html#comments
    http://www.c0t0d0s0.org/wfwcomment.php?cid=7144

    5
    http://www.c0t0d0s0.org/rss.php?version=2.0&type=comments&cid=7144
    

    nospam@example.com (Joerg Moellenkamp)
    Metabo cut saw for a month or so. When i saw the tag line on the box, i thought that this would be a nice tag line for Solaris: "Work. Don't play". What a pity that this tagline is already in use by a tool company  
    ]]>

    Wed, 19 Jan 2011 20:53:22 +0100
    http://www.c0t0d0s0.org/archives/7144-guid.html
    http://creativecommons.org/licenses/by-sa/2.5/
http://www.c0t0d0s0.org/archives/7144-Would-be-a-nice-tagline-for-Solaris-.....html

    
    http://feedproxy.google.com/~r/C0t0d0s0org-Solaris/~3/3VvbJ_TQMwc/7077-About-zonestat.html
            English
            Solaris
    
    http://www.c0t0d0s0.org/archives/7077-About-zonestat.html#comments
    http://www.c0t0d0s0.org/wfwcomment.php?cid=7077

    0
    http://www.c0t0d0s0.org/rss.php?version=2.0&type=comments&cid=7077
    

    nospam@example.com (Joerg Moellenkamp)
    a nice introduction to zonestat. Worth a read!  
    ]]>

    Tue, 07 Dec 2010 17:17:09 +0100
    http://www.c0t0d0s0.org/archives/7077-guid.html
    http://creativecommons.org/licenses/by-sa/2.5/
http://www.c0t0d0s0.org/archives/7077-About-zonestat.html