When I changed the ownership of the chef-server-api-10.16.2 gem dir to chef user, it worked. But I am still concerned as to why it would want to write there in the first place.
#
# Chef Server Config File
#
# Dynamically generated by Chef - local modifications will be replaced
log_level :info
log_location STDOUT
file_cache_path "/var/cache/chef"
sandbox_path "/var/cache/chef/sandboxes"
checksum_path "/var/lib/chef/cookbook_index"
node_path "/var/lib/chef/node"
cookbook_tarball_path "/var/lib/chef/cookbook-tarballs"
validation_client_name "chef-validator"
web_ui_admin_user_name "admin"
supportdir = "/var/lib/chef"
solr_jetty_path File.join(supportdir, "solr", "jetty")
solr_data_path File.join(supportdir, "solr", "data")
solr_home_path File.join(supportdir, "solr", "home")
solr_heap_size "256M"
umask 0022
signing_ca_cert "/etc/chef/certificates/cert.pem"
signing_ca_key "/etc/chef/certificates/key.pem"
signing_ca_user "chef"
signing_ca_group "chef"