From bc51a01453bd7c401870c2e3f81f74b340f8fcfc Mon Sep 17 00:00:00 2001 From: bran-wang Date: Mon, 12 Feb 2018 13:05:06 +0800 Subject: [PATCH 01/22] Fix trailing slash on kube/config failure #388 --- config/kube_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/kube_config.py b/config/kube_config.py index 4e09d6a95..85a72e63d 100644 --- a/config/kube_config.py +++ b/config/kube_config.py @@ -226,7 +226,7 @@ class KubeConfigLoader(object): def _load_cluster_info(self): if 'server' in self._cluster: - self.host = self._cluster['server'] + self.host = self._cluster['server'].rstrip('/') if self.host.startswith("https"): self.ssl_ca_cert = FileOrData( self._cluster, 'certificate-authority', From becae566343c48b9d8f4ee574fc97081ead9d847 Mon Sep 17 00:00:00 2001 From: Dov Reshef Date: Thu, 6 Sep 2018 12:28:50 +0300 Subject: [PATCH 02/22] Add partial support for out-of-tree client authentication providers (token only, no caching) --- config/exec_provider.py | 90 ++++++++++++++++++++++ config/exec_provider_test.py | 140 +++++++++++++++++++++++++++++++++++ config/kube_config.py | 28 +++++-- config/kube_config_test.py | 33 ++++++++- 4 files changed, 284 insertions(+), 7 deletions(-) create mode 100644 config/exec_provider.py create mode 100644 config/exec_provider_test.py diff --git a/config/exec_provider.py b/config/exec_provider.py new file mode 100644 index 000000000..9b8b645c4 --- /dev/null +++ b/config/exec_provider.py @@ -0,0 +1,90 @@ +# Copyright 2018 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import json +import os +import subprocess +import sys + +from .config_exception import ConfigException + + +class ExecProvider(object): + """ + Implementation of the proposal for out-of-tree client authentication providers + as described here -- + https://github.com/kubernetes/community/blob/master/contributors/design-proposals/auth/kubectl-exec-plugins.md + + Missing from implementation: + + * TLS cert support + * caching + """ + + def __init__(self, exec_config): + for key in ['command', 'apiVersion']: + if key not in exec_config: + raise ConfigException( + 'exec: malformed request. missing key \'%s\'' % key) + self.api_version = exec_config['apiVersion'] + self.args = [exec_config['command']] + if 'args' in exec_config: + self.args.extend(exec_config['args']) + self.env = os.environ.copy() + if 'env' in exec_config: + additional_vars = {} + for item in exec_config['env']: + name = item['name'] + value = item['value'] + additional_vars[name] = value + self.env.update(additional_vars) + + def run(self, previous_response=None): + kubernetes_exec_info = { + 'apiVersion': self.api_version, + 'kind': 'ExecCredential', + 'spec': { + 'interactive': sys.stdout.isatty() + } + } + if previous_response: + kubernetes_exec_info['spec']['response'] = previous_response + self.env['KUBERNETES_EXEC_INFO'] = json.dumps(kubernetes_exec_info) + process = subprocess.Popen( + self.args, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=self.env, + universal_newlines=True) + (stdout, stderr) = process.communicate() + exit_code = process.wait() + if exit_code != 0: + msg = 'exec: process returned %d' % exit_code + stderr = stderr.strip() + if stderr: + msg += '. %s' % stderr + raise ConfigException(msg) + try: + data = json.loads(stdout) + except ValueError as de: + raise ConfigException( + 'exec: failed to decode process output: %s' % de) + for key in ('apiVersion', 'kind', 'status'): + if key not in data: + raise ConfigException( + 'exec: malformed response. missing key \'%s\'' % key) + if data['apiVersion'] != self.api_version: + raise ConfigException( + 'exec: plugin api version %s does not match %s' % + (data['apiVersion'], self.api_version)) + return data['status'] diff --git a/config/exec_provider_test.py b/config/exec_provider_test.py new file mode 100644 index 000000000..a564e7660 --- /dev/null +++ b/config/exec_provider_test.py @@ -0,0 +1,140 @@ +# Copyright 2018 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import unittest + +import mock + +from .config_exception import ConfigException +from .exec_provider import ExecProvider + + +class ExecProviderTest(unittest.TestCase): + + def setUp(self): + self.input_ok = { + 'command': 'aws-iam-authenticator token -i dummy', + 'apiVersion': 'client.authentication.k8s.io/v1beta1' + } + self.output_ok = """ + { + "apiVersion": "client.authentication.k8s.io/v1beta1", + "kind": "ExecCredential", + "status": { + "token": "dummy" + } + } + """ + + def test_missing_input_keys(self): + exec_configs = [{}, {'command': ''}, {'apiVersion': ''}] + for exec_config in exec_configs: + with self.assertRaises(ConfigException) as context: + ExecProvider(exec_config) + self.assertIn('exec: malformed request. missing key', + context.exception.args[0]) + + @mock.patch('subprocess.Popen') + def test_error_code_returned(self, mock): + instance = mock.return_value + instance.wait.return_value = 1 + instance.communicate.return_value = ('', '') + with self.assertRaises(ConfigException) as context: + ep = ExecProvider(self.input_ok) + ep.run() + self.assertIn('exec: process returned %d' % + instance.wait.return_value, context.exception.args[0]) + + @mock.patch('subprocess.Popen') + def test_nonjson_output_returned(self, mock): + instance = mock.return_value + instance.wait.return_value = 0 + instance.communicate.return_value = ('', '') + with self.assertRaises(ConfigException) as context: + ep = ExecProvider(self.input_ok) + ep.run() + self.assertIn('exec: failed to decode process output', + context.exception.args[0]) + + @mock.patch('subprocess.Popen') + def test_missing_output_keys(self, mock): + instance = mock.return_value + instance.wait.return_value = 0 + outputs = [ + """ + { + "kind": "ExecCredential", + "status": { + "token": "dummy" + } + } + """, """ + { + "apiVersion": "client.authentication.k8s.io/v1beta1", + "status": { + "token": "dummy" + } + } + """, """ + { + "apiVersion": "client.authentication.k8s.io/v1beta1", + "kind": "ExecCredential" + } + """ + ] + for output in outputs: + instance.communicate.return_value = (output, '') + with self.assertRaises(ConfigException) as context: + ep = ExecProvider(self.input_ok) + ep.run() + self.assertIn('exec: malformed response. missing key', + context.exception.args[0]) + + @mock.patch('subprocess.Popen') + def test_mismatched_api_version(self, mock): + instance = mock.return_value + instance.wait.return_value = 0 + wrong_api_version = 'client.authentication.k8s.io/v1' + output = """ + { + "apiVersion": "%s", + "kind": "ExecCredential", + "status": { + "token": "dummy" + } + } + """ % wrong_api_version + instance.communicate.return_value = (output, '') + with self.assertRaises(ConfigException) as context: + ep = ExecProvider(self.input_ok) + ep.run() + self.assertIn( + 'exec: plugin api version %s does not match' % + wrong_api_version, + context.exception.args[0]) + + @mock.patch('subprocess.Popen') + def test_ok_01(self, mock): + instance = mock.return_value + instance.wait.return_value = 0 + instance.communicate.return_value = (self.output_ok, '') + ep = ExecProvider(self.input_ok) + result = ep.run() + self.assertTrue(isinstance(result, dict)) + self.assertTrue('token' in result) + + +if __name__ == '__main__': + unittest.main() diff --git a/config/kube_config.py b/config/kube_config.py index ddd3d02b0..671d370f7 100644 --- a/config/kube_config.py +++ b/config/kube_config.py @@ -1,4 +1,4 @@ -# Copyright 2016 The Kubernetes Authors. +# Copyright 2018 The Kubernetes Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -16,6 +16,7 @@ import atexit import base64 import datetime import json +import logging import os import tempfile import time @@ -30,6 +31,7 @@ from requests_oauthlib import OAuth2Session from six import PY3 from kubernetes.client import ApiClient, Configuration +from kubernetes.config.exec_provider import ExecProvider from .config_exception import ConfigException from .dateutil import UTC, format_rfc3339, parse_rfc3339 @@ -172,11 +174,10 @@ class KubeConfigLoader(object): section of kube-config and stops if it finds a valid authentication method. The order of authentication methods is: - 1. GCP auth-provider - 2. token_data - 3. token field (point to a token file) - 4. oidc auth-provider - 5. username/password + 1. auth-provider (gcp, azure, oidc) + 2. token field (point to a token file) + 3. exec provided plugin + 4. username/password """ if not self._user: return @@ -184,6 +185,8 @@ class KubeConfigLoader(object): return if self._load_user_token(): return + if self._load_from_exec_plugin(): + return self._load_user_pass_token() def _load_auth_provider_token(self): @@ -340,6 +343,19 @@ class KubeConfigLoader(object): provider['config'].value['id-token'] = refresh['id_token'] provider['config'].value['refresh-token'] = refresh['refresh_token'] + def _load_from_exec_plugin(self): + if 'exec' not in self._user: + return + try: + status = ExecProvider(self._user['exec']).run() + if 'token' not in status: + logging.error('exec: missing token field in plugin output') + return None + self.token = "Bearer %s" % status['token'] + return True + except Exception as e: + logging.error(str(e)) + def _load_user_token(self): token = FileOrData( self._user, 'tokenFile', 'token', diff --git a/config/kube_config_test.py b/config/kube_config_test.py index a79efb9a8..cd64f91bf 100644 --- a/config/kube_config_test.py +++ b/config/kube_config_test.py @@ -1,4 +1,4 @@ -# Copyright 2016 The Kubernetes Authors. +# Copyright 2018 The Kubernetes Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -422,6 +422,13 @@ class TestKubeConfigLoader(BaseTestCase): "user": "non_existing_user" } }, + { + "name": "exec_cred_user", + "context": { + "cluster": "default", + "user": "exec_cred_user" + } + }, ], "clusters": [ { @@ -573,6 +580,16 @@ class TestKubeConfigLoader(BaseTestCase): "client-key-data": TEST_CLIENT_KEY_BASE64, } }, + { + "name": "exec_cred_user", + "user": { + "exec": { + "apiVersion": "client.authentication.k8s.io/v1beta1", + "command": "aws-iam-authenticator", + "args": ["token", "-i", "dummy-cluster"] + } + } + }, ] } @@ -849,6 +866,20 @@ class TestKubeConfigLoader(BaseTestCase): active_context="non_existing_user").load_and_set(actual) self.assertEqual(expected, actual) + @mock.patch('kubernetes.config.kube_config.ExecProvider.run') + def test_user_exec_auth(self, mock): + token = "dummy" + mock.return_value = { + "token": token + } + expected = FakeConfig(host=TEST_HOST, api_key={ + "authorization": BEARER_TOKEN_FORMAT % token}) + actual = FakeConfig() + KubeConfigLoader( + config_dict=self.TEST_KUBE_CONFIG, + active_context="exec_cred_user").load_and_set(actual) + self.assertEqual(expected, actual) + if __name__ == '__main__': unittest.main() From 9d78cd794c3fc8f1fd2c5afbb8c3b2f051611986 Mon Sep 17 00:00:00 2001 From: Tomasz Prus Date: Tue, 18 Sep 2018 00:35:49 +0200 Subject: [PATCH 03/22] fix: read config data with bytes (python3) --- config/kube_config.py | 6 +++++- config/kube_config_test.py | 12 ++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/config/kube_config.py b/config/kube_config.py index 671d370f7..5e9c4ab19 100644 --- a/config/kube_config.py +++ b/config/kube_config.py @@ -100,8 +100,12 @@ class FileOrData(object): use_data_if_no_file = not self._file and self._data if use_data_if_no_file: if self._base64_file_content: + if isinstance(self._data, str): + content = self._data.encode() + else: + content = self._data self._file = _create_temp_file_with_content( - base64.decodestring(self._data.encode())) + base64.decodestring(content)) else: self._file = _create_temp_file_with_content(self._data) if self._file and not os.path.isfile(self._file): diff --git a/config/kube_config_test.py b/config/kube_config_test.py index cd64f91bf..84fb38aeb 100644 --- a/config/kube_config_test.py +++ b/config/kube_config_test.py @@ -201,6 +201,18 @@ class TestFileOrData(BaseTestCase): _create_temp_file_with_content(TEST_DATA))) _cleanup_temp_files() + def test_file_given_data_bytes(self): + obj = {TEST_DATA_KEY: TEST_DATA_BASE64.encode()} + t = FileOrData(obj=obj, file_key_name=TEST_FILE_KEY, + data_key_name=TEST_DATA_KEY) + self.assertEqual(TEST_DATA, self.get_file_content(t.as_file())) + + def test_file_given_data_bytes_no_base64(self): + obj = {TEST_DATA_KEY: TEST_DATA.encode()} + t = FileOrData(obj=obj, file_key_name=TEST_FILE_KEY, + data_key_name=TEST_DATA_KEY, base64_file_content=False) + self.assertEqual(TEST_DATA, self.get_file_content(t.as_file())) + class TestConfigNode(BaseTestCase): From 260d25793995da7fa6afec8a28621dbcd605000e Mon Sep 17 00:00:00 2001 From: Luiz Eduardo Date: Thu, 4 Oct 2018 12:08:42 +0200 Subject: [PATCH 04/22] Fix Issue-60: Replace encodestring and decodestring for standard_b64encode and standard_b64decode. --- config/kube_config.py | 4 ++-- config/kube_config_test.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/config/kube_config.py b/config/kube_config.py index 5e9c4ab19..a5396b956 100644 --- a/config/kube_config.py +++ b/config/kube_config.py @@ -105,7 +105,7 @@ class FileOrData(object): else: content = self._data self._file = _create_temp_file_with_content( - base64.decodestring(content)) + base64.standard_b64decode(content)) else: self._file = _create_temp_file_with_content(self._data) if self._file and not os.path.isfile(self._file): @@ -120,7 +120,7 @@ class FileOrData(object): with open(self._file) as f: if self._base64_file_content: self._data = bytes.decode( - base64.encodestring(str.encode(f.read()))) + base64.standard_b64encode(str.encode(f.read()))) else: self._data = f.read() return self._data diff --git a/config/kube_config_test.py b/config/kube_config_test.py index 84fb38aeb..7c9921ede 100644 --- a/config/kube_config_test.py +++ b/config/kube_config_test.py @@ -40,7 +40,7 @@ NON_EXISTING_FILE = "zz_non_existing_file_472398324" def _base64(string): - return base64.encodestring(string.encode()).decode() + return base64.standard_b64encode(string.encode()).decode() def _format_expiry_datetime(dt): From 3682e9b052498bbbac7cd805adaf7ad54212a64b Mon Sep 17 00:00:00 2001 From: Phil Hoffman Date: Thu, 4 Oct 2018 15:46:55 -0400 Subject: [PATCH 05/22] *Update ExecProvider to use safe_get() *Update unit tests to use ConfigNode() instead of dict() --- config/exec_provider.py | 9 +++++++-- config/exec_provider_test.py | 15 ++++++++++----- 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/config/exec_provider.py b/config/exec_provider.py index 9b8b645c4..436942f0f 100644 --- a/config/exec_provider.py +++ b/config/exec_provider.py @@ -32,16 +32,21 @@ class ExecProvider(object): """ def __init__(self, exec_config): + """ + exec_config must be of type ConfigNode because we depend on + safe_get(self, key) to correctly handle optional exec provider + config parameters. + """ for key in ['command', 'apiVersion']: if key not in exec_config: raise ConfigException( 'exec: malformed request. missing key \'%s\'' % key) self.api_version = exec_config['apiVersion'] self.args = [exec_config['command']] - if 'args' in exec_config: + if exec_config.safe_get('args'): self.args.extend(exec_config['args']) self.env = os.environ.copy() - if 'env' in exec_config: + if exec_config.safe_get('env'): additional_vars = {} for item in exec_config['env']: name = item['name'] diff --git a/config/exec_provider_test.py b/config/exec_provider_test.py index a564e7660..44579beb2 100644 --- a/config/exec_provider_test.py +++ b/config/exec_provider_test.py @@ -19,15 +19,18 @@ import mock from .config_exception import ConfigException from .exec_provider import ExecProvider +from .kube_config import ConfigNode class ExecProviderTest(unittest.TestCase): def setUp(self): - self.input_ok = { - 'command': 'aws-iam-authenticator token -i dummy', - 'apiVersion': 'client.authentication.k8s.io/v1beta1' - } + self.input_ok = ConfigNode('test', { + 'command': 'aws-iam-authenticator', + 'args': ['token', '-i', 'dummy'], + 'apiVersion': 'client.authentication.k8s.io/v1beta1', + 'env': None + }) self.output_ok = """ { "apiVersion": "client.authentication.k8s.io/v1beta1", @@ -39,7 +42,9 @@ class ExecProviderTest(unittest.TestCase): """ def test_missing_input_keys(self): - exec_configs = [{}, {'command': ''}, {'apiVersion': ''}] + exec_configs = [ConfigNode('test1', {}), + ConfigNode('test2', {'command': ''}), + ConfigNode('test3', {'apiVersion': ''})] for exec_config in exec_configs: with self.assertRaises(ConfigException) as context: ExecProvider(exec_config) From 13d57110144abda528cb9e61fbbf82e34a3992c8 Mon Sep 17 00:00:00 2001 From: micw523 Date: Sat, 27 Oct 2018 02:35:12 -0500 Subject: [PATCH 06/22] pep8 to pycodestyle --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 887d6647d..5d7f50f85 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,7 +10,7 @@ matrix: - python: 2.7 env: TOXENV=py27-functional - python: 2.7 - env: TOXENV=update-pep8 + env: TOXENV=update-pycodestyle - python: 2.7 env: TOXENV=docs - python: 2.7 From be621d3d329193faeba8c17ba9e8ffa036e59d5d Mon Sep 17 00:00:00 2001 From: micw523 Date: Fri, 2 Nov 2018 15:00:22 -0500 Subject: [PATCH 07/22] Fix for Travis CI failing on python-base --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 5d7f50f85..c3fefd02d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,6 @@ # ref: https://docs.travis-ci.com/user/languages/python language: python -dist: trusty +dist: xenial sudo: required matrix: From 8f3a69ea101d842887baab9389dcb42dc4ad8d9e Mon Sep 17 00:00:00 2001 From: Trevor Edwards Date: Fri, 5 Oct 2018 17:09:54 -0700 Subject: [PATCH 08/22] Refresh GCP tokens on retrieval by overriding client config method. --- config/kube_config.py | 16 +++++ config/kube_config_test.py | 121 ++++++++++++++++++++++++++++++++----- 2 files changed, 122 insertions(+), 15 deletions(-) diff --git a/config/kube_config.py b/config/kube_config.py index a5396b956..305b2e0ae 100644 --- a/config/kube_config.py +++ b/config/kube_config.py @@ -392,8 +392,24 @@ class KubeConfigLoader(object): if 'insecure-skip-tls-verify' in self._cluster: self.verify_ssl = not self._cluster['insecure-skip-tls-verify'] + def _using_gcp_auth_provider(self): + return self._user and \ + 'auth-provider' in self._user and \ + 'name' in self._user['auth-provider'] and \ + self._user['auth-provider']['name'] == 'gcp' + def _set_config(self, client_configuration): + if self._using_gcp_auth_provider(): + # GCP auth tokens must be refreshed regularly, but swagger expects + # a constant token. Replace the swagger-generated client config's + # get_api_key_with_prefix method with our own to allow automatic + # token refresh. + def _gcp_get_api_key(*args): + return self._load_gcp_token(self._user['auth-provider']) + client_configuration.get_api_key_with_prefix = _gcp_get_api_key if 'token' in self.__dict__: + # Note: this line runs for GCP auth tokens as well, but this entry + # will not be updated upon GCP token refresh. client_configuration.api_key['authorization'] = self.token # copy these keys directly from self to configuration object keys = ['host', 'ssl_ca_cert', 'cert_file', 'key_file', 'verify_ssl'] diff --git a/config/kube_config_test.py b/config/kube_config_test.py index 7c9921ede..ae9dc2255 100644 --- a/config/kube_config_test.py +++ b/config/kube_config_test.py @@ -24,6 +24,8 @@ import mock import yaml from six import PY3, next +from kubernetes.client import Configuration + from .config_exception import ConfigException from .kube_config import (ConfigNode, FileOrData, KubeConfigLoader, _cleanup_temp_files, _create_temp_file_with_content, @@ -34,7 +36,9 @@ BEARER_TOKEN_FORMAT = "Bearer %s" EXPIRY_DATETIME_FORMAT = "%Y-%m-%dT%H:%M:%SZ" # should be less than kube_config.EXPIRY_SKEW_PREVENTION_DELAY -EXPIRY_TIMEDELTA = 2 +PAST_EXPIRY_TIMEDELTA = 2 +# should be more than kube_config.EXPIRY_SKEW_PREVENTION_DELAY +FUTURE_EXPIRY_TIMEDELTA = 60 NON_EXISTING_FILE = "zz_non_existing_file_472398324" @@ -47,9 +51,9 @@ def _format_expiry_datetime(dt): return dt.strftime(EXPIRY_DATETIME_FORMAT) -def _get_expiry(loader): +def _get_expiry(loader, active_context): expired_gcp_conf = (item for item in loader._config.value.get("users") - if item.get("name") == "expired_gcp") + if item.get("name") == active_context) return next(expired_gcp_conf).get("user").get("auth-provider") \ .get("config").get("expiry") @@ -73,8 +77,11 @@ TEST_USERNAME = "me" TEST_PASSWORD = "pass" # token for me:pass TEST_BASIC_TOKEN = "Basic bWU6cGFzcw==" -TEST_TOKEN_EXPIRY = _format_expiry_datetime( - datetime.datetime.utcnow() - datetime.timedelta(minutes=EXPIRY_TIMEDELTA)) +DATETIME_EXPIRY_PAST = datetime.datetime.utcnow( +) - datetime.timedelta(minutes=PAST_EXPIRY_TIMEDELTA) +DATETIME_EXPIRY_FUTURE = datetime.datetime.utcnow( +) + datetime.timedelta(minutes=FUTURE_EXPIRY_TIMEDELTA) +TEST_TOKEN_EXPIRY_PAST = _format_expiry_datetime(DATETIME_EXPIRY_PAST) TEST_SSL_HOST = "https://test-host" TEST_CERTIFICATE_AUTH = "cert-auth" @@ -371,6 +378,13 @@ class TestKubeConfigLoader(BaseTestCase): "user": "expired_gcp" } }, + { + "name": "expired_gcp_refresh", + "context": { + "cluster": "default", + "user": "expired_gcp_refresh" + } + }, { "name": "oidc", "context": { @@ -509,7 +523,24 @@ class TestKubeConfigLoader(BaseTestCase): "name": "gcp", "config": { "access-token": TEST_DATA_BASE64, - "expiry": TEST_TOKEN_EXPIRY, # always in past + "expiry": TEST_TOKEN_EXPIRY_PAST, # always in past + } + }, + "token": TEST_DATA_BASE64, # should be ignored + "username": TEST_USERNAME, # should be ignored + "password": TEST_PASSWORD, # should be ignored + } + }, + # Duplicated from "expired_gcp" so test_load_gcp_token_with_refresh + # is isolated from test_gcp_get_api_key_with_prefix. + { + "name": "expired_gcp_refresh", + "user": { + "auth-provider": { + "name": "gcp", + "config": { + "access-token": TEST_DATA_BASE64, + "expiry": TEST_TOKEN_EXPIRY_PAST, # always in past } }, "token": TEST_DATA_BASE64, # should be ignored @@ -630,16 +661,20 @@ class TestKubeConfigLoader(BaseTestCase): self.assertEqual(BEARER_TOKEN_FORMAT % TEST_DATA_BASE64, loader.token) def test_gcp_no_refresh(self): - expected = FakeConfig( - host=TEST_HOST, - token=BEARER_TOKEN_FORMAT % TEST_DATA_BASE64) - actual = FakeConfig() + fake_config = FakeConfig() + # swagger-generated config has this, but FakeConfig does not. + self.assertFalse(hasattr(fake_config, 'get_api_key_with_prefix')) KubeConfigLoader( config_dict=self.TEST_KUBE_CONFIG, active_context="gcp", get_google_credentials=lambda: _raise_exception( - "SHOULD NOT BE CALLED")).load_and_set(actual) - self.assertEqual(expected, actual) + "SHOULD NOT BE CALLED")).load_and_set(fake_config) + # Should now be populated with a gcp token fetcher. + self.assertIsNotNone(fake_config.get_api_key_with_prefix) + self.assertEqual(TEST_HOST, fake_config.host) + # For backwards compatibility, authorization field should still be set. + self.assertEqual(BEARER_TOKEN_FORMAT % TEST_DATA_BASE64, + fake_config.api_key['authorization']) def test_load_gcp_token_no_refresh(self): loader = KubeConfigLoader( @@ -654,20 +689,48 @@ class TestKubeConfigLoader(BaseTestCase): def test_load_gcp_token_with_refresh(self): def cred(): return None cred.token = TEST_ANOTHER_DATA_BASE64 - cred.expiry = datetime.datetime.now() + cred.expiry = datetime.datetime.utcnow() loader = KubeConfigLoader( config_dict=self.TEST_KUBE_CONFIG, active_context="expired_gcp", get_google_credentials=lambda: cred) - original_expiry = _get_expiry(loader) + original_expiry = _get_expiry(loader, "expired_gcp") self.assertTrue(loader._load_auth_provider_token()) - new_expiry = _get_expiry(loader) + new_expiry = _get_expiry(loader, "expired_gcp") # assert that the configs expiry actually updates self.assertTrue(new_expiry > original_expiry) self.assertEqual(BEARER_TOKEN_FORMAT % TEST_ANOTHER_DATA_BASE64, loader.token) + def test_gcp_get_api_key_with_prefix(self): + class cred_old: + token = TEST_DATA_BASE64 + expiry = DATETIME_EXPIRY_PAST + + class cred_new: + token = TEST_ANOTHER_DATA_BASE64 + expiry = DATETIME_EXPIRY_FUTURE + fake_config = FakeConfig() + _get_google_credentials = mock.Mock() + _get_google_credentials.side_effect = [cred_old, cred_new] + + loader = KubeConfigLoader( + config_dict=self.TEST_KUBE_CONFIG, + active_context="expired_gcp_refresh", + get_google_credentials=_get_google_credentials) + loader.load_and_set(fake_config) + original_expiry = _get_expiry(loader, "expired_gcp_refresh") + # Call GCP token fetcher. + token = fake_config.get_api_key_with_prefix() + new_expiry = _get_expiry(loader, "expired_gcp_refresh") + + self.assertTrue(new_expiry > original_expiry) + self.assertEqual(BEARER_TOKEN_FORMAT % TEST_ANOTHER_DATA_BASE64, + loader.token) + self.assertEqual(BEARER_TOKEN_FORMAT % TEST_ANOTHER_DATA_BASE64, + token) + def test_oidc_no_refresh(self): loader = KubeConfigLoader( config_dict=self.TEST_KUBE_CONFIG, @@ -893,5 +956,33 @@ class TestKubeConfigLoader(BaseTestCase): self.assertEqual(expected, actual) +class TestKubernetesClientConfiguration(BaseTestCase): + # Verifies properties of kubernetes.client.Configuration. + # These tests guard against changes to the upstream configuration class, + # since GCP authorization overrides get_api_key_with_prefix to refresh its + # token regularly. + + def test_get_api_key_with_prefix_exists(self): + self.assertTrue(hasattr(Configuration, 'get_api_key_with_prefix')) + + def test_get_api_key_with_prefix_returns_token(self): + expected_token = 'expected_token' + config = Configuration() + config.api_key['authorization'] = expected_token + self.assertEqual(expected_token, + config.get_api_key_with_prefix('authorization')) + + def test_auth_settings_calls_get_api_key_with_prefix(self): + expected_token = 'expected_token' + + def fake_get_api_key_with_prefix(identifier): + self.assertEqual('authorization', identifier) + return expected_token + config = Configuration() + config.get_api_key_with_prefix = fake_get_api_key_with_prefix + self.assertEqual(expected_token, + config.auth_settings()['BearerToken']['value']) + + if __name__ == '__main__': unittest.main() From 2f3247b83715503daa9be353b8b45a8431e168ae Mon Sep 17 00:00:00 2001 From: micw523 Date: Fri, 9 Nov 2018 20:22:11 -0600 Subject: [PATCH 09/22] Travis CI for Python 3.7 --- .travis.yml | 4 ++++ tox.ini | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index c3fefd02d..18dcd2fd1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -25,6 +25,10 @@ matrix: env: TOXENV=py36 - python: 3.6 env: TOXENV=py36-functional + - python: 3.7 + env: TOXENV=py37 + - python: 3.7 + env: TOXENV=py37-functional install: - pip install tox diff --git a/tox.ini b/tox.ini index f36f34786..f935a6cd2 100644 --- a/tox.ini +++ b/tox.ini @@ -1,6 +1,6 @@ [tox] skipsdist = True -envlist = py27, py34, py35, py36 +envlist = py27, py34, py35, py36, py37 [testenv] passenv = TOXENV CI TRAVIS TRAVIS_* From 86ae2de36f56742d70e6caf6c15eda75a168aab6 Mon Sep 17 00:00:00 2001 From: Neha Yadav Date: Wed, 5 Dec 2018 22:22:10 +0530 Subject: [PATCH 10/22] Add verify-boilerplate script --- .travis.yml | 1 + hack/boilerplate/boilerplate.py | 197 ++++++++++++++++++++++++++++ hack/boilerplate/boilerplate.py.txt | 15 +++ hack/boilerplate/boilerplate.sh.txt | 13 ++ hack/verify-boilerplate.sh | 35 +++++ 5 files changed, 261 insertions(+) create mode 100755 hack/boilerplate/boilerplate.py create mode 100644 hack/boilerplate/boilerplate.py.txt create mode 100644 hack/boilerplate/boilerplate.sh.txt create mode 100755 hack/verify-boilerplate.sh diff --git a/.travis.yml b/.travis.yml index c3fefd02d..7aa0138b7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -31,4 +31,5 @@ install: script: - ./run_tox.sh tox + - ./hack/verify-boilerplate.sh diff --git a/hack/boilerplate/boilerplate.py b/hack/boilerplate/boilerplate.py new file mode 100755 index 000000000..bdc70c313 --- /dev/null +++ b/hack/boilerplate/boilerplate.py @@ -0,0 +1,197 @@ +#!/usr/bin/env python + +# Copyright 2018 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import print_function + +import argparse +import datetime +import difflib +import glob +import os +import re +import sys + +parser = argparse.ArgumentParser() +parser.add_argument( + "filenames", + help="list of files to check, all files if unspecified", + nargs='*') + +rootdir = os.path.dirname(__file__) + "/../../" +rootdir = os.path.abspath(rootdir) +parser.add_argument( + "--rootdir", default=rootdir, help="root directory to examine") + +default_boilerplate_dir = os.path.join(rootdir, "hack/boilerplate") +parser.add_argument( + "--boilerplate-dir", default=default_boilerplate_dir) + +parser.add_argument( + "-v", "--verbose", + help="give verbose output regarding why a file does not pass", + action="store_true") + +args = parser.parse_args() + +verbose_out = sys.stderr if args.verbose else open("/dev/null", "w") + + +def get_refs(): + refs = {} + + for path in glob.glob(os.path.join(args.boilerplate_dir, "boilerplate.*.txt")): + extension = os.path.basename(path).split(".")[1] + + ref_file = open(path, 'r') + ref = ref_file.read().splitlines() + ref_file.close() + refs[extension] = ref + + return refs + + +def file_passes(filename, refs, regexs): + try: + f = open(filename, 'r') + except Exception as exc: + print("Unable to open %s: %s" % (filename, exc), file=verbose_out) + return False + + data = f.read() + f.close() + + basename = os.path.basename(filename) + extension = file_extension(filename) + + if extension != "": + ref = refs[extension] + else: + ref = refs[basename] + + # remove extra content from the top of files + if extension == "sh": + p = regexs["shebang"] + (data, found) = p.subn("", data, 1) + + data = data.splitlines() + + # if our test file is smaller than the reference it surely fails! + if len(ref) > len(data): + print('File %s smaller than reference (%d < %d)' % + (filename, len(data), len(ref)), + file=verbose_out) + return False + + # trim our file to the same number of lines as the reference file + data = data[:len(ref)] + + p = regexs["year"] + for d in data: + if p.search(d): + print('File %s has the YEAR field, but missing the year of date' % + filename, file=verbose_out) + return False + + # Replace all occurrences of the regex "2014|2015|2016|2017|2018" with "YEAR" + p = regexs["date"] + for i, d in enumerate(data): + (data[i], found) = p.subn('YEAR', d) + if found != 0: + break + + # if we don't match the reference at this point, fail + if ref != data: + print("Header in %s does not match reference, diff:" % + filename, file=verbose_out) + if args.verbose: + print(file=verbose_out) + for line in difflib.unified_diff(ref, data, 'reference', filename, lineterm=''): + print(line, file=verbose_out) + print(file=verbose_out) + return False + + return True + + +def file_extension(filename): + return os.path.splitext(filename)[1].split(".")[-1].lower() + + +# list all the files contain 'DO NOT EDIT', but are not generated +skipped_ungenerated_files = ['hack/boilerplate/boilerplate.py'] + + +def normalize_files(files): + newfiles = [] + for pathname in files: + newfiles.append(pathname) + for i, pathname in enumerate(newfiles): + if not os.path.isabs(pathname): + newfiles[i] = os.path.join(args.rootdir, pathname) + return newfiles + + +def get_files(extensions): + files = [] + if len(args.filenames) > 0: + files = args.filenames + else: + for root, dirs, walkfiles in os.walk(args.rootdir): + for name in walkfiles: + pathname = os.path.join(root, name) + files.append(pathname) + + files = normalize_files(files) + outfiles = [] + for pathname in files: + basename = os.path.basename(pathname) + extension = file_extension(pathname) + if extension in extensions or basename in extensions: + outfiles.append(pathname) + return outfiles + + +def get_dates(): + years = datetime.datetime.now().year + return '(%s)' % '|'.join((str(year) for year in range(2014, years+1))) + + +def get_regexs(): + regexs = {} + # Search for "YEAR" which exists in the boilerplate, but shouldn't in the real thing + regexs["year"] = re.compile('YEAR') + # get_dates return 2014, 2015, 2016, 2017, or 2018 until the current year as a regex like: "(2014|2015|2016|2017|2018)"; + # company holder names can be anything + regexs["date"] = re.compile(get_dates()) + # strip #!.* from shell scripts + regexs["shebang"] = re.compile(r"^(#!.*\n)\n*", re.MULTILINE) + return regexs + + +def main(): + regexs = get_regexs() + refs = get_refs() + filenames = get_files(refs.keys()) + + for filename in filenames: + if not file_passes(filename, refs, regexs): + print(filename, file=sys.stdout) + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/hack/boilerplate/boilerplate.py.txt b/hack/boilerplate/boilerplate.py.txt new file mode 100644 index 000000000..d781daf9e --- /dev/null +++ b/hack/boilerplate/boilerplate.py.txt @@ -0,0 +1,15 @@ +#!/usr/bin/env python + +# Copyright YEAR The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/hack/boilerplate/boilerplate.sh.txt b/hack/boilerplate/boilerplate.sh.txt new file mode 100644 index 000000000..34cb349c4 --- /dev/null +++ b/hack/boilerplate/boilerplate.sh.txt @@ -0,0 +1,13 @@ +# Copyright YEAR The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/hack/verify-boilerplate.sh b/hack/verify-boilerplate.sh new file mode 100755 index 000000000..2f54c8cc3 --- /dev/null +++ b/hack/verify-boilerplate.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash + +# Copyright 2018 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +KUBE_ROOT=$(dirname "${BASH_SOURCE}")/.. + +boilerDir="${KUBE_ROOT}/hack/boilerplate" +boiler="${boilerDir}/boilerplate.py" + +files_need_boilerplate=($(${boiler} "$@")) + +# Run boilerplate check +if [[ ${#files_need_boilerplate[@]} -gt 0 ]]; then + for file in "${files_need_boilerplate[@]}"; do + echo "Boilerplate header is wrong for: ${file}" >&2 + done + + exit 1 +fi From d56fdbc0cc33a6c8e4782c93b50c56c889fb3fa3 Mon Sep 17 00:00:00 2001 From: Neha Yadav Date: Wed, 5 Dec 2018 22:22:59 +0530 Subject: [PATCH 11/22] Verify Boilerplate fix --- config/__init__.py | 2 ++ config/config_exception.py | 2 ++ config/dateutil.py | 2 ++ config/dateutil_test.py | 2 ++ config/exec_provider.py | 2 ++ config/exec_provider_test.py | 2 ++ config/incluster_config.py | 2 ++ config/incluster_config_test.py | 2 ++ config/kube_config.py | 2 ++ config/kube_config_test.py | 2 ++ run_tox.sh | 3 +-- stream/__init__.py | 2 ++ stream/stream.py | 20 ++++++++++++-------- stream/ws_client.py | 20 ++++++++++++-------- stream/ws_client_test.py | 4 +++- watch/__init__.py | 2 ++ watch/watch.py | 2 ++ watch/watch_test.py | 2 ++ 18 files changed, 56 insertions(+), 19 deletions(-) diff --git a/config/__init__.py b/config/__init__.py index 3476ff714..02a7532d5 100644 --- a/config/__init__.py +++ b/config/__init__.py @@ -1,3 +1,5 @@ +#!/usr/bin/env python + # Copyright 2016 The Kubernetes Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/config/config_exception.py b/config/config_exception.py index 23fab022c..9bf049c69 100644 --- a/config/config_exception.py +++ b/config/config_exception.py @@ -1,3 +1,5 @@ +#!/usr/bin/env python + # Copyright 2016 The Kubernetes Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/config/dateutil.py b/config/dateutil.py index ed88cba8b..402751cd2 100644 --- a/config/dateutil.py +++ b/config/dateutil.py @@ -1,3 +1,5 @@ +#!/usr/bin/env python + # Copyright 2017 The Kubernetes Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/config/dateutil_test.py b/config/dateutil_test.py index deb0ea880..7a13fad04 100644 --- a/config/dateutil_test.py +++ b/config/dateutil_test.py @@ -1,3 +1,5 @@ +#!/usr/bin/env python + # Copyright 2016 The Kubernetes Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/config/exec_provider.py b/config/exec_provider.py index 436942f0f..a41983539 100644 --- a/config/exec_provider.py +++ b/config/exec_provider.py @@ -1,3 +1,5 @@ +#!/usr/bin/env python + # Copyright 2018 The Kubernetes Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/config/exec_provider_test.py b/config/exec_provider_test.py index 44579beb2..8b6517b01 100644 --- a/config/exec_provider_test.py +++ b/config/exec_provider_test.py @@ -1,3 +1,5 @@ +#!/usr/bin/env python + # Copyright 2018 The Kubernetes Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/config/incluster_config.py b/config/incluster_config.py index 60fc0af82..e643f0df9 100644 --- a/config/incluster_config.py +++ b/config/incluster_config.py @@ -1,3 +1,5 @@ +#!/usr/bin/env python + # Copyright 2016 The Kubernetes Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/config/incluster_config_test.py b/config/incluster_config_test.py index 622b31b37..3cb0abfc8 100644 --- a/config/incluster_config_test.py +++ b/config/incluster_config_test.py @@ -1,3 +1,5 @@ +#!/usr/bin/env python + # Copyright 2016 The Kubernetes Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/config/kube_config.py b/config/kube_config.py index 958959e30..058ae290a 100644 --- a/config/kube_config.py +++ b/config/kube_config.py @@ -1,3 +1,5 @@ +#!/usr/bin/env python + # Copyright 2018 The Kubernetes Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/config/kube_config_test.py b/config/kube_config_test.py index ae9dc2255..ee4f49d9c 100644 --- a/config/kube_config_test.py +++ b/config/kube_config_test.py @@ -1,3 +1,5 @@ +#!/usr/bin/env python + # Copyright 2018 The Kubernetes Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/run_tox.sh b/run_tox.sh index 557337855..4b5839248 100755 --- a/run_tox.sh +++ b/run_tox.sh @@ -11,7 +11,7 @@ # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and +# See the License for the specific language governing permissions and # limitations under the License. set -o errexit @@ -51,4 +51,3 @@ git status echo "Running tox from the main repo on $TOXENV environment" # Run the user-provided command. "${@}" - diff --git a/stream/__init__.py b/stream/__init__.py index e72d05836..e9b7d24ff 100644 --- a/stream/__init__.py +++ b/stream/__init__.py @@ -1,3 +1,5 @@ +#!/usr/bin/env python + # Copyright 2017 The Kubernetes Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/stream/stream.py b/stream/stream.py index 0412fc338..3eab0b9ab 100644 --- a/stream/stream.py +++ b/stream/stream.py @@ -1,14 +1,18 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at +#!/usr/bin/env python + +# Copyright 2018 The Kubernetes Authors. # -# http://www.apache.org/licenses/LICENSE-2.0 +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. from . import ws_client diff --git a/stream/ws_client.py b/stream/ws_client.py index 1cc56cddc..c6fea7ba0 100644 --- a/stream/ws_client.py +++ b/stream/ws_client.py @@ -1,14 +1,18 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at +#!/usr/bin/env python + +# Copyright 2018 The Kubernetes Authors. # -# http://www.apache.org/licenses/LICENSE-2.0 +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. from kubernetes.client.rest import ApiException diff --git a/stream/ws_client_test.py b/stream/ws_client_test.py index e2eca96cc..756d95978 100644 --- a/stream/ws_client_test.py +++ b/stream/ws_client_test.py @@ -1,4 +1,6 @@ -# Copyright 2017 The Kubernetes Authors. +#!/usr/bin/env python + +# Copyright 2018 The Kubernetes Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/watch/__init__.py b/watch/__init__.py index ca9ac0698..46a31ceda 100644 --- a/watch/__init__.py +++ b/watch/__init__.py @@ -1,3 +1,5 @@ +#!/usr/bin/env python + # Copyright 2016 The Kubernetes Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/watch/watch.py b/watch/watch.py index 21899dd80..fb4c1abf8 100644 --- a/watch/watch.py +++ b/watch/watch.py @@ -1,3 +1,5 @@ +#!/usr/bin/env python + # Copyright 2016 The Kubernetes Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/watch/watch_test.py b/watch/watch_test.py index d1ec80a1c..f2804f4a3 100644 --- a/watch/watch_test.py +++ b/watch/watch_test.py @@ -1,3 +1,5 @@ +#!/usr/bin/env python + # Copyright 2016 The Kubernetes Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); From 375befb15cbbf418468d56554ee4b5de77232f3f Mon Sep 17 00:00:00 2001 From: Neha Yadav Date: Tue, 11 Dec 2018 22:46:45 +0530 Subject: [PATCH 12/22] Make dependancy adal optional --- config/kube_config.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/config/kube_config.py b/config/kube_config.py index 958959e30..e51697bcb 100644 --- a/config/kube_config.py +++ b/config/kube_config.py @@ -21,7 +21,6 @@ import os import tempfile import time -import adal import google.auth import google.auth.transport.requests import oauthlib.oauth2 @@ -36,6 +35,11 @@ from kubernetes.config.exec_provider import ExecProvider from .config_exception import ConfigException from .dateutil import UTC, format_rfc3339, parse_rfc3339 +try: + import adal +except ImportError: + pass + EXPIRY_SKEW_PREVENTION_DELAY = datetime.timedelta(minutes=5) KUBE_CONFIG_DEFAULT_LOCATION = os.environ.get('KUBECONFIG', '~/.kube/config') _temp_files = {} @@ -218,6 +222,9 @@ class KubeConfigLoader(object): return self.token def _refresh_azure_token(self, config): + if 'adal' not in globals(): + raise ImportError('refresh token error, adal library not imported') + tenant = config['tenant-id'] authority = 'https://login.microsoftonline.com/{}'.format(tenant) context = adal.AuthenticationContext( From ebb49d02ed90256cd002d1d75cb8a92125c4392e Mon Sep 17 00:00:00 2001 From: Xavier Vello Date: Mon, 7 Jan 2019 18:19:57 +0100 Subject: [PATCH 13/22] Use safe_load and safe_dump for all yaml calls --- config/kube_config.py | 2 +- config/kube_config_test.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/config/kube_config.py b/config/kube_config.py index 958959e30..300d924e0 100644 --- a/config/kube_config.py +++ b/config/kube_config.py @@ -493,7 +493,7 @@ class ConfigNode(object): def _get_kube_config_loader_for_yaml_file(filename, **kwargs): with open(filename) as f: return KubeConfigLoader( - config_dict=yaml.load(f), + config_dict=yaml.safe_load(f), config_base_path=os.path.abspath(os.path.dirname(filename)), **kwargs) diff --git a/config/kube_config_test.py b/config/kube_config_test.py index ae9dc2255..f0bddf8ba 100644 --- a/config/kube_config_test.py +++ b/config/kube_config_test.py @@ -896,14 +896,14 @@ class TestKubeConfigLoader(BaseTestCase): def test_load_kube_config(self): expected = FakeConfig(host=TEST_HOST, token=BEARER_TOKEN_FORMAT % TEST_DATA_BASE64) - config_file = self._create_temp_file(yaml.dump(self.TEST_KUBE_CONFIG)) + config_file = self._create_temp_file(yaml.safe_dump(self.TEST_KUBE_CONFIG)) actual = FakeConfig() load_kube_config(config_file=config_file, context="simple_token", client_configuration=actual) self.assertEqual(expected, actual) def test_list_kube_config_contexts(self): - config_file = self._create_temp_file(yaml.dump(self.TEST_KUBE_CONFIG)) + config_file = self._create_temp_file(yaml.safe_dump(self.TEST_KUBE_CONFIG)) contexts, active_context = list_kube_config_contexts( config_file=config_file) self.assertDictEqual(self.TEST_KUBE_CONFIG['contexts'][0], @@ -916,7 +916,7 @@ class TestKubeConfigLoader(BaseTestCase): contexts) def test_new_client_from_config(self): - config_file = self._create_temp_file(yaml.dump(self.TEST_KUBE_CONFIG)) + config_file = self._create_temp_file(yaml.safe_dump(self.TEST_KUBE_CONFIG)) client = new_client_from_config( config_file=config_file, context="simple_token") self.assertEqual(TEST_HOST, client.configuration.host) From 13ff5184ac43c0bffa813bbba4fca04d610c45d7 Mon Sep 17 00:00:00 2001 From: Xavier Vello Date: Tue, 8 Jan 2019 10:37:28 +0100 Subject: [PATCH 14/22] linting --- config/kube_config_test.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/config/kube_config_test.py b/config/kube_config_test.py index f0bddf8ba..37ff3e27c 100644 --- a/config/kube_config_test.py +++ b/config/kube_config_test.py @@ -896,14 +896,16 @@ class TestKubeConfigLoader(BaseTestCase): def test_load_kube_config(self): expected = FakeConfig(host=TEST_HOST, token=BEARER_TOKEN_FORMAT % TEST_DATA_BASE64) - config_file = self._create_temp_file(yaml.safe_dump(self.TEST_KUBE_CONFIG)) + config_file = self._create_temp_file( + yaml.safe_dump(self.TEST_KUBE_CONFIG)) actual = FakeConfig() load_kube_config(config_file=config_file, context="simple_token", client_configuration=actual) self.assertEqual(expected, actual) def test_list_kube_config_contexts(self): - config_file = self._create_temp_file(yaml.safe_dump(self.TEST_KUBE_CONFIG)) + config_file = self._create_temp_file( + yaml.safe_dump(self.TEST_KUBE_CONFIG)) contexts, active_context = list_kube_config_contexts( config_file=config_file) self.assertDictEqual(self.TEST_KUBE_CONFIG['contexts'][0], @@ -916,7 +918,8 @@ class TestKubeConfigLoader(BaseTestCase): contexts) def test_new_client_from_config(self): - config_file = self._create_temp_file(yaml.safe_dump(self.TEST_KUBE_CONFIG)) + config_file = self._create_temp_file( + yaml.safe_dump(self.TEST_KUBE_CONFIG)) client = new_client_from_config( config_file=config_file, context="simple_token") self.assertEqual(TEST_HOST, client.configuration.host) From 3c30a3099336a5976074c18ea61814646689b4a8 Mon Sep 17 00:00:00 2001 From: Julian Taylor Date: Sat, 19 Jan 2019 12:38:57 +0100 Subject: [PATCH 15/22] fix watching with a specified resource version The watch code reset the version to the last found in the response. When you first list existing objects and then start watching from that resource version the existing versions are older than the version you wanted and the watch starts from the wrong version after the first restart. This leads to for example already deleted objects ending in the stream again. Fix this by setting the minimum resource version to reset from to the input resource version. As long as k8s returns all objects in order in the watch this should work. We cannot use the integer value of the resource version to order it as one should be treat the value as opaque. Closes https://github.com/kubernetes-client/python/issues/700 --- watch/watch.py | 2 ++ watch/watch_test.py | 73 ++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 74 insertions(+), 1 deletion(-) diff --git a/watch/watch.py b/watch/watch.py index 21899dd80..a9c315cd0 100644 --- a/watch/watch.py +++ b/watch/watch.py @@ -122,6 +122,8 @@ class Watch(object): return_type = self.get_return_type(func) kwargs['watch'] = True kwargs['_preload_content'] = False + if 'resource_version' in kwargs: + self.resource_version = kwargs['resource_version'] timeouts = ('timeout_seconds' in kwargs) while True: diff --git a/watch/watch_test.py b/watch/watch_test.py index d1ec80a1c..672c0526a 100644 --- a/watch/watch_test.py +++ b/watch/watch_test.py @@ -14,12 +14,15 @@ import unittest -from mock import Mock +from mock import Mock, call from .watch import Watch class WatchTests(unittest.TestCase): + def setUp(self): + # counter for a test that needs test global state + self.callcount = 0 def test_watch_with_decode(self): fake_resp = Mock() @@ -62,6 +65,74 @@ class WatchTests(unittest.TestCase): fake_resp.close.assert_called_once() fake_resp.release_conn.assert_called_once() + def test_watch_resource_version_set(self): + # https://github.com/kubernetes-client/python/issues/700 + # ensure watching from a resource version does reset to resource + # version 0 after k8s resets the watch connection + fake_resp = Mock() + fake_resp.close = Mock() + fake_resp.release_conn = Mock() + values = [ + '{"type": "ADDED", "object": {"metadata": {"name": "test1",' + '"resourceVersion": "1"}, "spec": {}, "status": {}}}\n', + '{"type": "ADDED", "object": {"metadata": {"name": "test2",' + '"resourceVersion": "2"}, "spec": {}, "sta', + 'tus": {}}}\n' + '{"type": "ADDED", "object": {"metadata": {"name": "test3",' + '"resourceVersion": "3"}, "spec": {}, "status": {}}}\n' + ] + # return nothing on the first call and values on the second + # this emulates a watch from a rv that returns nothing in the first k8s + # watch reset and values later + + def get_values(*args, **kwargs): + self.callcount += 1 + if self.callcount == 1: + return [] + else: + return values + + fake_resp.read_chunked = Mock( + side_effect=get_values) + + fake_api = Mock() + fake_api.get_namespaces = Mock(return_value=fake_resp) + fake_api.get_namespaces.__doc__ = ':return: V1NamespaceList' + + w = Watch() + # ensure we keep our requested resource version or the version latest + # returned version when the existing versions are older than the + # requested version + # needed for the list existing objects, then watch from there use case + calls = [] + + iterations = 2 + # first two calls must use the passed rv, the first call is a + # "reset" and does not actually return anything + # the second call must use the same rv but will return values + # (with a wrong rv but a real cluster would behave correctly) + # calls following that will use the rv from those returned values + calls.append(call(_preload_content=False, watch=True, + resource_version="5")) + calls.append(call(_preload_content=False, watch=True, + resource_version="5")) + for i in range(iterations): + # ideally we want 5 here but as rv must be treated as an + # opaque value we cannot interpret it and order it so rely + # on k8s returning the events completely and in order + calls.append(call(_preload_content=False, watch=True, + resource_version="3")) + + for c, e in enumerate(w.stream(fake_api.get_namespaces, + resource_version="5")): + if c == len(values) * iterations: + w.stop() + + # check calls are in the list, gives good error output + fake_api.get_namespaces.assert_has_calls(calls) + # more strict test with worse error message + self.assertEqual(fake_api.get_namespaces.mock_calls, calls) + def test_watch_stream_twice(self): w = Watch(float) for step in ['first', 'second']: From 4d387d5879ab280ecf18ffb0b39846b040fd533b Mon Sep 17 00:00:00 2001 From: Roy Lenferink Date: Mon, 4 Feb 2019 19:01:16 +0100 Subject: [PATCH 16/22] Updated OWNERS to include link to docs --- OWNERS | 2 ++ 1 file changed, 2 insertions(+) diff --git a/OWNERS b/OWNERS index 7a860ad20..cfec4b11e 100644 --- a/OWNERS +++ b/OWNERS @@ -1,3 +1,5 @@ +# See the OWNERS docs at https://go.k8s.io/owners + approvers: - mbohlool - yliaog From 0fc0d404acd4a6080409e2796b7f6d6002039861 Mon Sep 17 00:00:00 2001 From: Neha Yadav Date: Fri, 8 Feb 2019 02:46:07 +0530 Subject: [PATCH 17/22] Update pycodestyle --- config/exec_provider.py | 7 ++++--- config/incluster_config.py | 3 ++- config/kube_config.py | 6 ++++-- hack/boilerplate/boilerplate.py | 14 +++++++++----- stream/ws_client.py | 7 ++++--- 5 files changed, 23 insertions(+), 14 deletions(-) diff --git a/config/exec_provider.py b/config/exec_provider.py index a41983539..89d81e8c4 100644 --- a/config/exec_provider.py +++ b/config/exec_provider.py @@ -23,9 +23,10 @@ from .config_exception import ConfigException class ExecProvider(object): """ - Implementation of the proposal for out-of-tree client authentication providers - as described here -- - https://github.com/kubernetes/community/blob/master/contributors/design-proposals/auth/kubectl-exec-plugins.md + Implementation of the proposal for out-of-tree client + authentication providers as described here -- + https://github.com/kubernetes/community/blob/master/contributors + /design-proposals/auth/kubectl-exec-plugins.md Missing from implementation: diff --git a/config/incluster_config.py b/config/incluster_config.py index e643f0df9..c9bdc907d 100644 --- a/config/incluster_config.py +++ b/config/incluster_config.py @@ -87,7 +87,8 @@ class InClusterConfigLoader(object): def load_incluster_config(): - """Use the service account kubernetes gives to pods to connect to kubernetes + """ + Use the service account kubernetes gives to pods to connect to kubernetes cluster. It's intended for clients that expect to be running inside a pod running on kubernetes. It will raise an exception if called from a process not running in a kubernetes environment.""" diff --git a/config/kube_config.py b/config/kube_config.py index c0e0e26d8..743046dbd 100644 --- a/config/kube_config.py +++ b/config/kube_config.py @@ -556,9 +556,11 @@ def new_client_from_config( config_file=None, context=None, persist_config=True): - """Loads configuration the same as load_kube_config but returns an ApiClient + """ + Loads configuration the same as load_kube_config but returns an ApiClient to be used with any API object. This will allow the caller to concurrently - talk with multiple clusters.""" + talk with multiple clusters. + """ client_config = type.__call__(Configuration) load_kube_config(config_file=config_file, context=context, client_configuration=client_config, diff --git a/hack/boilerplate/boilerplate.py b/hack/boilerplate/boilerplate.py index bdc70c313..61d4cb947 100755 --- a/hack/boilerplate/boilerplate.py +++ b/hack/boilerplate/boilerplate.py @@ -52,7 +52,8 @@ verbose_out = sys.stderr if args.verbose else open("/dev/null", "w") def get_refs(): refs = {} - for path in glob.glob(os.path.join(args.boilerplate_dir, "boilerplate.*.txt")): + for path in glob.glob(os.path.join( + args.boilerplate_dir, "boilerplate.*.txt")): extension = os.path.basename(path).split(".")[1] ref_file = open(path, 'r') @@ -105,7 +106,7 @@ def file_passes(filename, refs, regexs): filename, file=verbose_out) return False - # Replace all occurrences of the regex "2014|2015|2016|2017|2018" with "YEAR" + # Replace all occurrences of regex "2014|2015|2016|2017|2018" with "YEAR" p = regexs["date"] for i, d in enumerate(data): (data[i], found) = p.subn('YEAR', d) @@ -118,7 +119,8 @@ def file_passes(filename, refs, regexs): filename, file=verbose_out) if args.verbose: print(file=verbose_out) - for line in difflib.unified_diff(ref, data, 'reference', filename, lineterm=''): + for line in difflib.unified_diff( + ref, data, 'reference', filename, lineterm=''): print(line, file=verbose_out) print(file=verbose_out) return False @@ -171,9 +173,11 @@ def get_dates(): def get_regexs(): regexs = {} - # Search for "YEAR" which exists in the boilerplate, but shouldn't in the real thing + # Search for "YEAR" which exists in the boilerplate, + # but shouldn't in the real thing regexs["year"] = re.compile('YEAR') - # get_dates return 2014, 2015, 2016, 2017, or 2018 until the current year as a regex like: "(2014|2015|2016|2017|2018)"; + # get_dates return 2014, 2015, 2016, 2017, or 2018 until the current year + # as a regex like: "(2014|2015|2016|2017|2018)"; # company holder names can be anything regexs["date"] = re.compile(get_dates()) # strip #!.* from shell scripts diff --git a/stream/ws_client.py b/stream/ws_client.py index c6fea7ba0..cf8a3fe99 100644 --- a/stream/ws_client.py +++ b/stream/ws_client.py @@ -53,7 +53,8 @@ class WSClient: header.append("authorization: %s" % headers['authorization']) if headers and 'sec-websocket-protocol' in headers: - header.append("sec-websocket-protocol: %s" % headers['sec-websocket-protocol']) + header.append("sec-websocket-protocol: %s" % + headers['sec-websocket-protocol']) else: header.append("sec-websocket-protocol: v4.channel.k8s.io") @@ -186,8 +187,8 @@ class WSClient: data = data[1:] if data: if channel in [STDOUT_CHANNEL, STDERR_CHANNEL]: - # keeping all messages in the order they received for - # non-blocking call. + # keeping all messages in the order they received + # for non-blocking call. self._all += data if channel not in self._channels: self._channels[channel] = data From 0229f0adb26951e82bd9fb3ef7344951c52e4b75 Mon Sep 17 00:00:00 2001 From: micw523 Date: Mon, 11 Feb 2019 17:11:37 -0600 Subject: [PATCH 18/22] Restore one-line link --- config/exec_provider.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/config/exec_provider.py b/config/exec_provider.py index 89d81e8c4..a0348f1e9 100644 --- a/config/exec_provider.py +++ b/config/exec_provider.py @@ -25,8 +25,7 @@ class ExecProvider(object): """ Implementation of the proposal for out-of-tree client authentication providers as described here -- - https://github.com/kubernetes/community/blob/master/contributors - /design-proposals/auth/kubectl-exec-plugins.md + https://github.com/kubernetes/community/blob/master/contributors/design-proposals/auth/kubectl-exec-plugins.md Missing from implementation: From 8e6f0435a38e24aac700d9ebac700bdf6138ba8c Mon Sep 17 00:00:00 2001 From: Mitar Date: Mon, 15 Oct 2018 23:57:46 -0700 Subject: [PATCH 19/22] Making watch work with read_namespaced_pod_log. Fixes https://github.com/kubernetes-client/python/issues/199. --- watch/watch.py | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/watch/watch.py b/watch/watch.py index bdf24f1ab..79b2358d7 100644 --- a/watch/watch.py +++ b/watch/watch.py @@ -20,6 +20,7 @@ import pydoc from kubernetes import client PYDOC_RETURN_LABEL = ":return:" +PYDOC_FOLLOW_PARAM = ":param bool follow:" # Removing this suffix from return type name should give us event's object # type. e.g., if list_namespaces() returns "NamespaceList" type, @@ -65,7 +66,7 @@ class Watch(object): self._raw_return_type = return_type self._stop = False self._api_client = client.ApiClient() - self.resource_version = 0 + self.resource_version = None def stop(self): self._stop = True @@ -78,8 +79,17 @@ class Watch(object): return return_type[:-len(TYPE_LIST_SUFFIX)] return return_type + def get_watch_argument_name(self, func): + if PYDOC_FOLLOW_PARAM in pydoc.getdoc(func): + return 'follow' + else: + return 'watch' + def unmarshal_event(self, data, return_type): - js = json.loads(data) + try: + js = json.loads(data) + except ValueError: + return data js['raw_object'] = js['object'] if return_type: obj = SimpleNamespace(data=json.dumps(js['raw_object'])) @@ -122,7 +132,7 @@ class Watch(object): self._stop = False return_type = self.get_return_type(func) - kwargs['watch'] = True + kwargs[self.get_watch_argument_name(func)] = True kwargs['_preload_content'] = False if 'resource_version' in kwargs: self.resource_version = kwargs['resource_version'] @@ -136,9 +146,12 @@ class Watch(object): if self._stop: break finally: - kwargs['resource_version'] = self.resource_version resp.close() resp.release_conn() + if self.resource_version is not None: + kwargs['resource_version'] = self.resource_version + else: + break if timeouts or self._stop: break From ad06e5c923b2d4e5db86f7e91deddb95a6dc9a43 Mon Sep 17 00:00:00 2001 From: Mitar Date: Mon, 18 Feb 2019 16:43:50 -0800 Subject: [PATCH 20/22] Added tests. --- watch/watch_test.py | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/watch/watch_test.py b/watch/watch_test.py index 08eb36c21..ebc400af4 100644 --- a/watch/watch_test.py +++ b/watch/watch_test.py @@ -67,6 +67,35 @@ class WatchTests(unittest.TestCase): fake_resp.close.assert_called_once() fake_resp.release_conn.assert_called_once() + def test_watch_for_follow(self): + fake_resp = Mock() + fake_resp.close = Mock() + fake_resp.release_conn = Mock() + fake_resp.read_chunked = Mock( + return_value=[ + 'log_line_1\n', + 'log_line_2\n']) + + fake_api = Mock() + fake_api.read_namespaced_pod_log = Mock(return_value=fake_resp) + fake_api.read_namespaced_pod_log.__doc__ = ':param bool follow:\n:return: str' + + w = Watch() + count = 1 + for e in w.stream(fake_api.read_namespaced_pod_log): + self.assertEqual("log_line_1", e) + count += 1 + # make sure we can stop the watch and the last event with won't be + # returned + if count == 2: + w.stop() + + fake_api.read_namespaced_pod_log.assert_called_once_with( + _preload_content=False, follow=True) + fake_resp.read_chunked.assert_called_once_with(decode_content=False) + fake_resp.close.assert_called_once() + fake_resp.release_conn.assert_called_once() + def test_watch_resource_version_set(self): # https://github.com/kubernetes-client/python/issues/700 # ensure watching from a resource version does reset to resource From 972a76a83d0133b45db03495b0f9fd05ed2b94a3 Mon Sep 17 00:00:00 2001 From: Mitar Date: Wed, 20 Feb 2019 23:56:38 -0800 Subject: [PATCH 21/22] Don't use break inside finally. It swallows exceptions. --- watch/watch.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/watch/watch.py b/watch/watch.py index 79b2358d7..5966eaceb 100644 --- a/watch/watch.py +++ b/watch/watch.py @@ -151,7 +151,7 @@ class Watch(object): if self.resource_version is not None: kwargs['resource_version'] = self.resource_version else: - break + self._stop = True if timeouts or self._stop: break From 328b2d12452c9125fa74590e971423970c1d750a Mon Sep 17 00:00:00 2001 From: Tomasz Prus Date: Sat, 20 Oct 2018 00:49:51 +0200 Subject: [PATCH 22/22] feat: merging kubeconfig files --- config/kube_config.py | 134 ++++++++++++++++++++++++------ config/kube_config_test.py | 165 ++++++++++++++++++++++++++++++++++++- 2 files changed, 274 insertions(+), 25 deletions(-) diff --git a/config/kube_config.py b/config/kube_config.py index 300d924e0..be6156cb6 100644 --- a/config/kube_config.py +++ b/config/kube_config.py @@ -14,10 +14,12 @@ import atexit import base64 +import copy import datetime import json import logging import os +import platform import tempfile import time @@ -38,6 +40,7 @@ from .dateutil import UTC, format_rfc3339, parse_rfc3339 EXPIRY_SKEW_PREVENTION_DELAY = datetime.timedelta(minutes=5) KUBE_CONFIG_DEFAULT_LOCATION = os.environ.get('KUBECONFIG', '~/.kube/config') +ENV_KUBECONFIG_PATH_SEPARATOR = ';' if platform.system() == 'Windows' else ':' _temp_files = {} @@ -132,7 +135,12 @@ class KubeConfigLoader(object): get_google_credentials=None, config_base_path="", config_persister=None): - self._config = ConfigNode('kube-config', config_dict) + + if isinstance(config_dict, ConfigNode): + self._config = config_dict + else: + self._config = ConfigNode('kube-config', config_dict) + self._current_context = None self._user = None self._cluster = None @@ -361,9 +369,10 @@ class KubeConfigLoader(object): logging.error(str(e)) def _load_user_token(self): + base_path = self._get_base_path(self._user.path) token = FileOrData( self._user, 'tokenFile', 'token', - file_base_path=self._config_base_path, + file_base_path=base_path, base64_file_content=False).as_data() if token: self.token = "Bearer %s" % token @@ -376,19 +385,27 @@ class KubeConfigLoader(object): self._user['password'])).get('authorization') return True + def _get_base_path(self, config_path): + if self._config_base_path is not None: + return self._config_base_path + if config_path is not None: + return os.path.abspath(os.path.dirname(config_path)) + return "" + def _load_cluster_info(self): if 'server' in self._cluster: self.host = self._cluster['server'].rstrip('/') if self.host.startswith("https"): + base_path = self._get_base_path(self._cluster.path) self.ssl_ca_cert = FileOrData( self._cluster, 'certificate-authority', - file_base_path=self._config_base_path).as_file() + file_base_path=base_path).as_file() self.cert_file = FileOrData( self._user, 'client-certificate', - file_base_path=self._config_base_path).as_file() + file_base_path=base_path).as_file() self.key_file = FileOrData( self._user, 'client-key', - file_base_path=self._config_base_path).as_file() + file_base_path=base_path).as_file() if 'insecure-skip-tls-verify' in self._cluster: self.verify_ssl = not self._cluster['insecure-skip-tls-verify'] @@ -435,9 +452,10 @@ class ConfigNode(object): message in case of missing keys. The assumption is all access keys are present in a well-formed kube-config.""" - def __init__(self, name, value): + def __init__(self, name, value, path=None): self.name = name self.value = value + self.path = path def __contains__(self, key): return key in self.value @@ -457,7 +475,7 @@ class ConfigNode(object): 'Invalid kube-config file. Expected key %s in %s' % (key, self.name)) if isinstance(v, dict) or isinstance(v, list): - return ConfigNode('%s/%s' % (self.name, key), v) + return ConfigNode('%s/%s' % (self.name, key), v, self.path) else: return v @@ -482,7 +500,12 @@ class ConfigNode(object): 'Expected only one object with name %s in %s list' % (name, self.name)) if result is not None: - return ConfigNode('%s[name=%s]' % (self.name, name), result) + if isinstance(result, ConfigNode): + return result + else: + return ConfigNode( + '%s[name=%s]' % + (self.name, name), result, self.path) if safe: return None raise ConfigException( @@ -490,18 +513,87 @@ class ConfigNode(object): 'Expected object with name %s in %s list' % (name, self.name)) -def _get_kube_config_loader_for_yaml_file(filename, **kwargs): - with open(filename) as f: - return KubeConfigLoader( - config_dict=yaml.safe_load(f), - config_base_path=os.path.abspath(os.path.dirname(filename)), - **kwargs) +class KubeConfigMerger: + + """Reads and merges configuration from one or more kube-config's. + The propery `config` can be passed to the KubeConfigLoader as config_dict. + + It uses a path attribute from ConfigNode to store the path to kubeconfig. + This path is required to load certs from relative paths. + + A method `save_changes` updates changed kubeconfig's (it compares current + state of dicts with). + """ + + def __init__(self, paths): + self.paths = [] + self.config_files = {} + self.config_merged = None + + for path in paths.split(ENV_KUBECONFIG_PATH_SEPARATOR): + if path: + path = os.path.expanduser(path) + if os.path.exists(path): + self.paths.append(path) + self.load_config(path) + self.config_saved = copy.deepcopy(self.config_files) + + @property + def config(self): + return self.config_merged + + def load_config(self, path): + with open(path) as f: + config = yaml.safe_load(f) + + if self.config_merged is None: + config_merged = copy.deepcopy(config) + for item in ('clusters', 'contexts', 'users'): + config_merged[item] = [] + self.config_merged = ConfigNode(path, config_merged, path) + + for item in ('clusters', 'contexts', 'users'): + self._merge(item, config[item], path) + self.config_files[path] = config + + def _merge(self, item, add_cfg, path): + for new_item in add_cfg: + for exists in self.config_merged.value[item]: + if exists['name'] == new_item['name']: + break + else: + self.config_merged.value[item].append(ConfigNode( + '{}/{}'.format(path, new_item), new_item, path)) + + def save_changes(self): + for path in self.paths: + if self.config_saved[path] != self.config_files[path]: + self.save_config(path) + self.config_saved = copy.deepcopy(self.config_files) + + def save_config(self, path): + with open(path, 'w') as f: + yaml.safe_dump(self.config_files[path], f, + default_flow_style=False) + + +def _get_kube_config_loader_for_yaml_file( + filename, persist_config=False, **kwargs): + + kcfg = KubeConfigMerger(filename) + if persist_config and 'config_persister' not in kwargs: + kwargs['config_persister'] = kcfg.save_changes() + + return KubeConfigLoader( + config_dict=kcfg.config, + config_base_path=None, + **kwargs) def list_kube_config_contexts(config_file=None): if config_file is None: - config_file = os.path.expanduser(KUBE_CONFIG_DEFAULT_LOCATION) + config_file = KUBE_CONFIG_DEFAULT_LOCATION loader = _get_kube_config_loader_for_yaml_file(config_file) return loader.list_contexts(), loader.current_context @@ -523,18 +615,12 @@ def load_kube_config(config_file=None, context=None, """ if config_file is None: - config_file = os.path.expanduser(KUBE_CONFIG_DEFAULT_LOCATION) - - config_persister = None - if persist_config: - def _save_kube_config(config_map): - with open(config_file, 'w') as f: - yaml.safe_dump(config_map, f, default_flow_style=False) - config_persister = _save_kube_config + config_file = KUBE_CONFIG_DEFAULT_LOCATION loader = _get_kube_config_loader_for_yaml_file( config_file, active_context=context, - config_persister=config_persister) + persist_config=persist_config) + if client_configuration is None: config = type.__call__(Configuration) loader.load_and_set(config) diff --git a/config/kube_config_test.py b/config/kube_config_test.py index 37ff3e27c..dc783c21b 100644 --- a/config/kube_config_test.py +++ b/config/kube_config_test.py @@ -27,7 +27,8 @@ from six import PY3, next from kubernetes.client import Configuration from .config_exception import ConfigException -from .kube_config import (ConfigNode, FileOrData, KubeConfigLoader, +from .kube_config import (ENV_KUBECONFIG_PATH_SEPARATOR, ConfigNode, + FileOrData, KubeConfigLoader, KubeConfigMerger, _cleanup_temp_files, _create_temp_file_with_content, list_kube_config_contexts, load_kube_config, new_client_from_config) @@ -987,5 +988,167 @@ class TestKubernetesClientConfiguration(BaseTestCase): config.auth_settings()['BearerToken']['value']) +class TestKubeConfigMerger(BaseTestCase): + TEST_KUBE_CONFIG_PART1 = { + "current-context": "no_user", + "contexts": [ + { + "name": "no_user", + "context": { + "cluster": "default" + } + }, + ], + "clusters": [ + { + "name": "default", + "cluster": { + "server": TEST_HOST + } + }, + ], + "users": [] + } + + TEST_KUBE_CONFIG_PART2 = { + "current-context": "", + "contexts": [ + { + "name": "ssl", + "context": { + "cluster": "ssl", + "user": "ssl" + } + }, + { + "name": "simple_token", + "context": { + "cluster": "default", + "user": "simple_token" + } + }, + ], + "clusters": [ + { + "name": "ssl", + "cluster": { + "server": TEST_SSL_HOST, + "certificate-authority-data": + TEST_CERTIFICATE_AUTH_BASE64, + } + }, + ], + "users": [ + { + "name": "ssl", + "user": { + "token": TEST_DATA_BASE64, + "client-certificate-data": TEST_CLIENT_CERT_BASE64, + "client-key-data": TEST_CLIENT_KEY_BASE64, + } + }, + ] + } + + TEST_KUBE_CONFIG_PART3 = { + "current-context": "no_user", + "contexts": [ + { + "name": "expired_oidc", + "context": { + "cluster": "default", + "user": "expired_oidc" + } + }, + { + "name": "ssl", + "context": { + "cluster": "skipped-part2-defined-this-context", + "user": "skipped" + } + }, + ], + "clusters": [ + ], + "users": [ + { + "name": "expired_oidc", + "user": { + "auth-provider": { + "name": "oidc", + "config": { + "client-id": "tectonic-kubectl", + "client-secret": "FAKE_SECRET", + "id-token": TEST_OIDC_EXPIRED_LOGIN, + "idp-certificate-authority-data": TEST_OIDC_CA, + "idp-issuer-url": "https://example.org/identity", + "refresh-token": + "lucWJjEhlxZW01cXI3YmVlcYnpxNGhzk" + } + } + } + }, + { + "name": "simple_token", + "user": { + "token": TEST_DATA_BASE64, + "username": TEST_USERNAME, # should be ignored + "password": TEST_PASSWORD, # should be ignored + } + }, + ] + } + + def _create_multi_config(self): + files = [] + for part in ( + self.TEST_KUBE_CONFIG_PART1, + self.TEST_KUBE_CONFIG_PART2, + self.TEST_KUBE_CONFIG_PART3): + files.append(self._create_temp_file(yaml.safe_dump(part))) + return ENV_KUBECONFIG_PATH_SEPARATOR.join(files) + + def test_list_kube_config_contexts(self): + kubeconfigs = self._create_multi_config() + expected_contexts = [ + {'context': {'cluster': 'default'}, 'name': 'no_user'}, + {'context': {'cluster': 'ssl', 'user': 'ssl'}, 'name': 'ssl'}, + {'context': {'cluster': 'default', 'user': 'simple_token'}, + 'name': 'simple_token'}, + {'context': {'cluster': 'default', 'user': 'expired_oidc'}, 'name': 'expired_oidc'}] + + contexts, active_context = list_kube_config_contexts( + config_file=kubeconfigs) + + self.assertEqual(contexts, expected_contexts) + self.assertEqual(active_context, expected_contexts[0]) + + def test_new_client_from_config(self): + kubeconfigs = self._create_multi_config() + client = new_client_from_config( + config_file=kubeconfigs, context="simple_token") + self.assertEqual(TEST_HOST, client.configuration.host) + self.assertEqual(BEARER_TOKEN_FORMAT % TEST_DATA_BASE64, + client.configuration.api_key['authorization']) + + def test_save_changes(self): + kubeconfigs = self._create_multi_config() + + # load configuration, update token, save config + kconf = KubeConfigMerger(kubeconfigs) + user = kconf.config['users'].get_with_name('expired_oidc')['user'] + provider = user['auth-provider']['config'] + provider.value['id-token'] = "token-changed" + kconf.save_changes() + + # re-read configuration + kconf = KubeConfigMerger(kubeconfigs) + user = kconf.config['users'].get_with_name('expired_oidc')['user'] + provider = user['auth-provider']['config'] + + # new token + self.assertEqual(provider.value['id-token'], "token-changed") + + if __name__ == '__main__': unittest.main()