[chef] chef_env getting set incorrectly in node attr data

hi. i'm trying to figure out how the chef_environment is being set
incorrectly on some of my nodes, and could use advice on how to rummage
around to find the culprit.

one of my recipes relies on grabbing something from a data bag depending
on which chef_environment is set. i'm currently seeing a "dev" item
being grabbed on a host which should be "prod".

as background, the way i bootstrap chef onto my systems, hence establishing
at the very start which chef_env it is, is by doing:

# tail -1 /etc/chef/client.rb
environment "prod"
# chef-client -j /etc/chef/first-boot.json -l debug -L 
/var/log/chef/client.log

# knife environment show dev | grep ^name
name:                 dev
# knife environment show prod | grep ^name
name:                 prod

(in case you want to see fuller output of the above, it's at the end.)

in viewing this node's attr data, i can see there are 2 places where
chef_environment is set. the correct and desired env is "prod", which
appears at the end of the node attr data below. the incorrect env is
"dev", which appears as a "normal" attr at the top of the below output.

one thing i tried was to use "knife node edit". i'm not certain of what
the -E arg is supposed to do..

# knife node edit -E prod admin1.prod.spergatronic.com

the node attr json appeared in $EDITOR, i corrected the chef_env value
under the "normal" attrs. i ran chef-client again on this client, and
then the normal attrs again showed the wrong chef_env - "dev".

fwiw, in the client debug output, i do see the likes of
[Thu, 26 Jan 2012 23:45:31 +0000] DEBUG: Sending HTTP Request via GET to 
chef.dev.spergatronic.com:443/environments/prod

any halps? many thanks,
kallen


chef server: ubuntu 11.04, 0.10.8-1
chef client: centos  5.7,  0.10.4-1

the following output was created by
# knife node show admin1.prod.spergatronic.com -l -Fj

for brevity, i'm snipped out a lot, but left in the relevant bits.

{
  "normal": {
    "mail_type": "master",
    "chef_environment": "dev",
    "tags": [

    ],
    "aws": {
      "aws_x509_cert_path": "/etc/ec2/certs/servercert.pem",
      "aws_x509_key_path": "/etc/ec2/certs/privatekey.pem"
    },
  },
  "name": "admin1.prod.spergatronic.com",
  "override": {
    "chef_client": {
      "server_url": "https://chef.dev.spergatronic.com";,
      "init_style": "init",
      "validation_client_name": "chef-validator",
      "runstyle": "cron"
    },
    "postfix": {
      "smtp_sasl_auth_enable": "yes",
      "mail_type": "master",
      "mail_relay_networks": "10.0.0.0/8, 127.0.0.0/8, 172.22.22.0/24"
    }
  },
  "default": {
    "aws_x509_cert_path": "/etc/ec2/certs/servercert.pem",
    },
    "firewall": {
      "rules": [

      ],
      "securitylevel": ""
    },
    "chef_server_url": "https://chef.dev.spergatronic.com";,
    "aws_x509_key_path": "/etc/aws/certs/privatekey.pem",
    "yum": {
      "exclude": {
      },
      "installonlypkgs": {
      },
      "ius_release": "1.0-8",
      "epel_release": "5-4"
    },
    "aws": {
      "right_aws_version": "2.1.0"
    },
    "s3cmd": {
      "users": [
        "root"
      ]
    },
    "resolver": {
      "domain": [
        "prod.spergatronic.com"
      ],
      "nameservers": [
        "10.251.x.x",
        "10.64.x.x",
        "10.252.x.x",
        "10.245.x.x"
      ],
      "search": [
        "prod.spergatronic.com"
      ]
    },
    "chef_client": {
      "conf_dir": "/etc/chef",
      "cache_path": "/var/cache/chef",
      "backup_path": "/var/lib/chef",
      "init_style": "init",
      "ohai_plugin_disable": true,
      "validation_client_name": "chef-validator",
      "run_path": "/var/run/chef",
      "bin": "/usr/bin/chef-client",
      "splay": "20",
      "interval": "1800",
      "croninterval": "0 */4 * * *",
      "log_dir": "/var/log/chef"
    },
    "postfix": {
      "myorigin": "$myhostname",
      "myhostname": "admin1.prod.spergatronic.com",
      "relayhost": "[outbound.mailhop.org]",
      "smtp_fallback_relay": "[smtp.gmail.com]:587",
      "smtp_use_tls": "yes",
    }
  },
  "automatic": {
    "dmi": {
      "dmidecode_version": "2.11"
    },
    "keys": {
      "ssh": {
        "host_dsa_public": "AAAABsnip",
        "host_rsa_public": "AAAABsnip",
      }
    },
    "kernel": {
snip
      },
      "machine": "i686",
      "name": "Linux",
      "os": "GNU/Linux",
      "version": "#1 SMP Fri Feb 15 12:39:36 EST 2008",
      "release": "2.6.21.7-2.fc8xen"
    },
    "platform_version": "5.7",
    "fqdn": "admin1.prod.spergatronic.com",
    "command": {
      "ps": "ps -ef"
    },
    "ipaddress": "10.215.x.x",
    "domain": "prod.spergatronic.com",
    "os": "linux",
    "lsb": {
    },
    "virtualization": {
      "role": "guest",
      "system": "xen"
    },
    "current_user": "root",
    "ohai_time": 1327618833.9791,
    "chef_packages": {
      "ohai": {
        "ohai_root": "/usr/lib/ruby/gems/1.8/gems/ohai-0.6.4/lib/ohai",
        "version": "0.6.4"
      },
      "chef": {
        "version": "0.10.4",
        "chef_root": "/usr/lib/ruby/gems/1.8/gems/chef-0.10.4/lib"
      }
    },
    "recipes": [
      "chef-client::delete_validation",
      "chef-client::config",
      "chef-client",
      "aws",
      "toolbin",
      "s3cmd",
      "users::ops",
      "yum",
      "bind9::nameserver",
      "syslog-ng::17syslog-ng",
      "postfix",
      "postfix::sasl_auth"
    ],
    "os_version": "2.6.21.7-2.fc8xen",
    "languages": {
      "python": {
        "version": "2.4.3",
        "builddate": "Sep 21 2011, 20:06:00"
      },
      "perl": {
        "version": "5.8.8",
        "archname": "i386-linux-thread-multi"
      },
      "ruby": {
        "target_os": "linux",
        "bin_dir": "/usr/bin",
        "host_vendor": "redhat",
        "target_vendor": "redhat",
        "target_cpu": "i386",
        "ruby_bin": "/usr/bin/ruby",
        "version": "1.8.7",
        "host_os": "linux-gnu",
        "target": "i386-redhat-linux-gnu",
        "release_date": "2011-06-30",
        "host_cpu": "i686",
        "host": "i686-redhat-linux-gnu",
        "gems_dir": "/usr/lib/ruby/gems/1.8",
        "platform": "i386-linux"
      }
    },
    "uptime": "49 days 03 hours 03 minutes 44 seconds",
    "hostname": "admin1",
    "roles": [
      "admin-server",
      "syslog-ng-server",
      "postfix-server",
      "dns-client",
      "chef-client",
      "vpn-server",
      "dns-server",
      "base"
    ],
    "ec2": {
      "placement_availability_zone": "us-east-1a",
    },
    "platform": "centos",
  },
  "chef_environment": "prod",
  "run_list": [
    "role[base]",
    "role[dns-server]",
    "role[dns-client]",
    "role[syslog-ng-server]",
    "role[postfix-server]",
    "role[admin-server]"
  ]
}




[chef-repo]$ knife environment show dev
chef_type:            environment
cookbook_versions:    
default_attributes:  
  resolver: 
    domain:       dev.spergatronic.com
    nameservers: 
        10.69.xx.xx
        10.66.89.xx
        10.xx.55.xx
        10.xx.65.xx
    search:       dev.spergatronic.com
description:          Development environment (dev)
json_class:           Chef::Environment
name:                 dev
override_attributes:  


[chef-repo]$ knife environment show prod
chef_type:            environment
cookbook_versions:    
default_attributes:  
  resolver: 
    domain:       prod.spergatronic.com
    nameservers: 
        10.251.xx.0
        10.64.xx.51
        10.252.xx.163
        10.245.xx.162
    search:       prod.spergatronic.com
description:          Production environment (prod)
json_class:           Chef::Environment
name:                 prod
override_attributes: