From baf04dd46ddac263ff16d762d8c99f088a6e167c Mon Sep 17 00:00:00 2001 From: kkb0318 Date: Tue, 21 May 2024 21:10:16 +0900 Subject: [PATCH] impl irsa controller --- api/v1alpha1/irsa_types.go | 4 + internal/aws/aws_role.go | 7 +- internal/controller/irsa_controller.go | 107 +++++++++- internal/controller/irsa_controller_test.go | 199 ++++++++++++++---- .../controller/irsasetup_controller_test.go | 31 ++- internal/manifests/serviceaccount.go | 9 +- 6 files changed, 290 insertions(+), 67 deletions(-) diff --git a/api/v1alpha1/irsa_types.go b/api/v1alpha1/irsa_types.go index f890f42..0e004e5 100644 --- a/api/v1alpha1/irsa_types.go +++ b/api/v1alpha1/irsa_types.go @@ -22,6 +22,10 @@ import ( // IRSASpec defines the desired state of IRSA type IRSASpec struct { + // Cleanup, when enabled, allows the IRSA to perform garbage collection + // of resources that are no longer needed or managed. + // +required + Cleanup bool `json:"cleanup"` // ServiceAccount represents the Kubernetes service account associated with the IRSA ServiceAccount IRSAServiceAccount `json:"serviceAccount,omitempty"` // IamRole represents the IAM role details associated with the IRSA diff --git a/internal/aws/aws_role.go b/internal/aws/aws_role.go index 538cef8..2bb0ee3 100644 --- a/internal/aws/aws_role.go +++ b/internal/aws/aws_role.go @@ -22,6 +22,9 @@ type RoleManager struct { Namespaces []string // Policies represents the list of policies to be attached to the role Policies []string + + // AccountId represents the AWS Account Id + AccountId string } func (r *RoleManager) PolicyArn(policy string) *string { @@ -58,8 +61,8 @@ func (a *AwsIamClient) DeleteIRSARole(ctx context.Context, r RoleManager) error } // CreateIRSARole creates an IAM role with the specified trust policy and attaches specified policies to it -func (a *AwsIamClient) CreateIRSARole(ctx context.Context, accountId, issuerHostPath string, r RoleManager) error { - providerArn := fmt.Sprintf("arn:aws:iam::%s:oidc-provider/%s", accountId, issuerHostPath) +func (a *AwsIamClient) CreateIRSARole(ctx context.Context, issuerHostPath string, r RoleManager) error { + providerArn := fmt.Sprintf("arn:aws:iam::%s:oidc-provider/%s", r.AccountId, issuerHostPath) statement := make([]map[string]interface{}, len(r.Namespaces)) for i, ns := range r.Namespaces { statement[i] = map[string]interface{}{ diff --git a/internal/controller/irsa_controller.go b/internal/controller/irsa_controller.go index bef8ece..89b412b 100644 --- a/internal/controller/irsa_controller.go +++ b/internal/controller/irsa_controller.go @@ -19,10 +19,16 @@ package controller import ( "context" + awsclient "github.com/kkb0318/irsa-manager/internal/aws" + "github.com/kkb0318/irsa-manager/internal/handler" + "github.com/kkb0318/irsa-manager/internal/kubernetes" + "github.com/kkb0318/irsa-manager/internal/manifests" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + ctrllog "sigs.k8s.io/controller-runtime/pkg/log" irsav1alpha1 "github.com/kkb0318/irsa-manager/api/v1alpha1" ) @@ -30,7 +36,8 @@ import ( // IRSAReconciler reconciles a IRSA object type IRSAReconciler struct { client.Client - Scheme *runtime.Scheme + Scheme *runtime.Scheme + AwsClient awsclient.AwsClient } //+kubebuilder:rbac:groups=irsa.kkb0318.github.io,resources=irsas,verbs=get;list;watch;create;update;patch;delete @@ -47,12 +54,106 @@ type IRSAReconciler struct { // For more details, check Reconcile and its Result here: // - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.16.3/pkg/reconcile func (r *IRSAReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { - _ = log.FromContext(ctx) + log := ctrllog.FromContext(ctx) + obj := &irsav1alpha1.IRSA{} + if err := r.Get(ctx, req.NamespacedName, obj); err != nil { + return ctrl.Result{}, client.IgnoreNotFound(err) + } + if r.AwsClient == nil { + awsClient, err := awsclient.NewAwsClientFactory(ctx) + if err != nil { + return ctrl.Result{}, err + } + r.AwsClient = awsClient + } + kubeClient, err := kubernetes.NewKubernetesClient(r.Client, kubernetes.Owner{Field: "irsa-manager"}) + if err != nil { + return ctrl.Result{}, err + } + if !controllerutil.ContainsFinalizer(obj, irsamanagerFinalizer) { + controllerutil.AddFinalizer(obj, irsamanagerFinalizer) + if err := r.Update(ctx, obj); err != nil { + log.Error(err, "Failed to update custom resource to add finalizer") + return ctrl.Result{}, err + } + return ctrl.Result{Requeue: true}, nil + } + + defer func() { + if err := r.Get(ctx, req.NamespacedName, &irsav1alpha1.IRSA{}); err != nil { + return + } + statusHandler := handler.NewStatusHandler(kubeClient) + if err := statusHandler.Patch(ctx, obj); err != nil { + return + } + }() + + if !obj.DeletionTimestamp.IsZero() { + err = r.reconcileDelete(ctx, obj, kubeClient) + if err != nil { + return ctrl.Result{}, err + } + controllerutil.RemoveFinalizer(obj, irsamanagerFinalizer) + err = r.Update(ctx, obj) + if err == nil { + log.Info("successfully deleted") + } + return ctrl.Result{}, err + } + + if err := r.reconcile(ctx, obj, kubeClient); err != nil { + return ctrl.Result{}, err + } // roleArn: arn:aws:iam::{accountId}:role/{roleName} + log.Info("successfully reconciled") return ctrl.Result{}, nil } +func (r *IRSAReconciler) reconcileDelete(ctx context.Context, obj *irsav1alpha1.IRSA, kubeClient *kubernetes.KubernetesClient) error { + if !obj.Spec.Cleanup { + return nil + } + return nil + // kubeHandler := handler.NewKubernetesHandler(kubeClient) + // kubeHandler.Append(secret) + // err := kubeHandler.DeleteAll(ctx) + // if err != nil { + // return err + // } + // return selfhosted.Delete(ctx, factory) +} + +func (r *IRSAReconciler) reconcile(ctx context.Context, obj *irsav1alpha1.IRSA, kubeClient *kubernetes.KubernetesClient) error { + serviceAccount := obj.Spec.ServiceAccount + accountId, err := r.AwsClient.StsClient().GetAccountId() + if err != nil { + return err + } + roleManager := awsclient.RoleManager{ + RoleName: obj.Spec.IamRole.Name, + Namespaces: serviceAccount.Namespaces, + Policies: obj.Spec.IamPolicies, + AccountId: accountId, + } + err = r.AwsClient.IamClient().CreateIRSARole(ctx, "", roleManager) + if err != nil { + return err + } + kubeHandler := handler.NewKubernetesHandler(kubeClient) + + for _, ns := range serviceAccount.Namespaces { + sa := manifests.NewServiceAccountBuilder().WithIRSAAnnotation(roleManager).Build(types.NamespacedName{ + Name: serviceAccount.Name, + Namespace: ns, + }) + kubeHandler.Append(sa) + + } + return kubeHandler.ApplyAll(ctx) +} + // SetupWithManager sets up the controller with the Manager. func (r *IRSAReconciler) SetupWithManager(mgr ctrl.Manager) error { return ctrl.NewControllerManagedBy(mgr). diff --git a/internal/controller/irsa_controller_test.go b/internal/controller/irsa_controller_test.go index ec2c5cb..d3c3422 100644 --- a/internal/controller/irsa_controller_test.go +++ b/internal/controller/irsa_controller_test.go @@ -17,7 +17,7 @@ limitations under the License. package controller import ( - "context" + "fmt" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -31,54 +31,165 @@ import ( ) var _ = Describe("IRSA Controller", func() { - Context("When reconciling a resource", func() { - const resourceName = "test-resource" - - ctx := context.Background() - - typeNamespacedName := types.NamespacedName{ - Name: resourceName, - Namespace: "default", // TODO(user):Modify as needed - } - irsa := &irsav1alpha1.IRSA{} - - BeforeEach(func() { - By("creating the custom resource for the Kind IRSA") - err := k8sClient.Get(ctx, typeNamespacedName, irsa) - if err != nil && errors.IsNotFound(err) { - resource := &irsav1alpha1.IRSA{ + Context("When reconciling IRSA", func() { + tests := []struct { + name string + objs *irsav1alpha1.IRSA + f func(*IRSAReconciler, *irsav1alpha1.IRSA) + }{ + { + name: "should reconcile successfully", + objs: &irsav1alpha1.IRSA{ ObjectMeta: metav1.ObjectMeta{ - Name: resourceName, + Name: "test-resource1", Namespace: "default", }, - // TODO(user): Specify other spec details if needed. + Spec: irsav1alpha1.IRSASpec{ + ServiceAccount: irsav1alpha1.IRSAServiceAccount{ + Name: "sa-1", + Namespaces: []string{ + "kube-system", + "default", + }, + }, + }, + }, + f: func(r *IRSAReconciler, obj *irsav1alpha1.IRSA) { + expected := []expectedResource{ + { + NamespacedName: types.NamespacedName{Name: "sa-1", Namespace: "kube-system"}, + f: newServiceAccount, + }, + { + NamespacedName: types.NamespacedName{Name: "sa-1", Namespace: "default"}, + f: newServiceAccount, + }, + } + + By("Reconciling the created resource") + typeNamespacedName := types.NamespacedName{ + Name: obj.Name, + Namespace: obj.Namespace, + } + + _, err := r.Reconcile(ctx, reconcile.Request{ + NamespacedName: typeNamespacedName, + }) + Expect(err).NotTo(HaveOccurred()) + _, err = r.Reconcile(ctx, reconcile.Request{ + NamespacedName: typeNamespacedName, + }) + Expect(err).NotTo(HaveOccurred()) + for _, expect := range expected { + checkExist(expect) + } + By("removing the custom resource for the Kind") + Eventually(func() error { + return k8sClient.Delete(ctx, obj) + }, timeout).Should(Succeed()) + _, err = r.Reconcile(ctx, reconcile.Request{ + NamespacedName: typeNamespacedName, + }) + Expect(err).To(Not(HaveOccurred())) + for _, expect := range expected { + checkNoExist(expect) + } + }, + }, + { + name: "error", + objs: &irsav1alpha1.IRSA{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-resource2", + Namespace: "default", + }, + Spec: irsav1alpha1.IRSASpec{ + ServiceAccount: irsav1alpha1.IRSAServiceAccount{ + Name: "sa-2", + Namespaces: []string{ + "kube-system", + "default", + }, + }, + }, + }, + f: func(r *IRSAReconciler, obj *irsav1alpha1.IRSA) { + expected := []expectedResource{ + { + NamespacedName: types.NamespacedName{Name: "sa-1", Namespace: "kube-system"}, + f: newServiceAccount, + }, + { + NamespacedName: types.NamespacedName{Name: "sa-1", Namespace: "default"}, + f: newServiceAccount, + }, + } + + By("Reconciling the created resource") + typeNamespacedName := types.NamespacedName{ + Name: obj.Name, + Namespace: obj.Namespace, + } + + By("Error when creating role") + r.AwsClient = newMockAwsClient(&mockAwsIamAPI{createRoleErr: fmt.Errorf("createRoleErr")}, nil, nil) + _, err := r.Reconcile(ctx, reconcile.Request{ + NamespacedName: typeNamespacedName, + }) + Expect(err).To(HaveOccurred()) + for _, expect := range expected { + checkNoExist(expect) + } + + By("successfully Reconciling") + r.AwsClient = newMockAwsClient(&mockAwsIamAPI{}, nil, nil) + _, err = r.Reconcile(ctx, reconcile.Request{ + NamespacedName: typeNamespacedName, + }) + Expect(err).NotTo(HaveOccurred()) + for _, expect := range expected { + checkExist(expect) + } + By("removing the custom resource for the Kind") + Eventually(func() error { + return k8sClient.Delete(ctx, obj) + }, timeout).Should(Succeed()) + _, err = r.Reconcile(ctx, reconcile.Request{ + NamespacedName: typeNamespacedName, + }) + Expect(err).To(Not(HaveOccurred())) + for _, expect := range expected { + checkNoExist(expect) + } + }, + }, + } + for _, tt := range tests { + It(tt.name, func() { + typeNamespacedName := types.NamespacedName{ + Name: tt.objs.Name, + Namespace: tt.objs.Namespace, } - Expect(k8sClient.Create(ctx, resource)).To(Succeed()) - } - }) - - AfterEach(func() { - // TODO(user): Cleanup logic after each test, like removing the resource instance. - resource := &irsav1alpha1.IRSA{} - err := k8sClient.Get(ctx, typeNamespacedName, resource) - Expect(err).NotTo(HaveOccurred()) - - By("Cleanup the specific resource instance IRSA") - Expect(k8sClient.Delete(ctx, resource)).To(Succeed()) - }) - It("should successfully reconcile the resource", func() { - By("Reconciling the created resource") - controllerReconciler := &IRSAReconciler{ - Client: k8sClient, - Scheme: k8sClient.Scheme(), - } - - _, err := controllerReconciler.Reconcile(ctx, reconcile.Request{ - NamespacedName: typeNamespacedName, + controllerReconciler := &IRSAReconciler{ + Client: k8sClient, + Scheme: k8sClient.Scheme(), + AwsClient: newMockAwsClient(&mockAwsIamAPI{}, nil, nil), + } + By("creating the custom resource for the Kind IRSASetup") + err := k8sClient.Get(ctx, typeNamespacedName, tt.objs) + if err != nil && errors.IsNotFound(err) { + Expect(k8sClient.Create(ctx, tt.objs)).To(Succeed()) + } + _, err = controllerReconciler.Reconcile(ctx, reconcile.Request{ + NamespacedName: typeNamespacedName, + }) + Expect(err).NotTo(HaveOccurred()) + tt.f(controllerReconciler, tt.objs) }) - Expect(err).NotTo(HaveOccurred()) - // TODO(user): Add more specific assertions depending on your controller's reconciliation logic. - // Example: If you expect a certain status condition after reconciliation, verify it here. + } + BeforeEach(func() { + }) + AfterEach(func() { }) }) }) diff --git a/internal/controller/irsasetup_controller_test.go b/internal/controller/irsasetup_controller_test.go index 5006009..a581d95 100644 --- a/internal/controller/irsasetup_controller_test.go +++ b/internal/controller/irsasetup_controller_test.go @@ -180,7 +180,7 @@ var _ = Describe("IRSASetup Controller", func() { Namespace: obj.Namespace, } By("secret does not exist when reconciling with the AwsClient error") - r.AwsClient = newMockAwsClient(&mockAwsIamAPI{createOidcErr: true}, &mockAwsS3API{}, &mockAwsStsAPI{}) + r.AwsClient = newMockAwsClient(&mockAwsIamAPI{createOidcErr: fmt.Errorf("createOidcErr")}, &mockAwsS3API{}, &mockAwsStsAPI{}) _, err := r.Reconcile(ctx, reconcile.Request{ NamespacedName: typeNamespacedName, }) @@ -347,8 +347,13 @@ func (m *mockAwsClient) StsClient() *awsclient.AwsStsClient { type ( mockAwsIamAPI struct { - createOidcErr bool - deleteOidcErr bool + createOidcErr error + deleteOidcErr error + createRoleErr error + deleteRoleErr error + updateAssumeRolePolicyError error + attachRolePolicyError error + detachRolePolicyError error } mockAwsS3API struct { createBucketErr bool @@ -358,37 +363,31 @@ type ( ) func (m *mockAwsIamAPI) CreateOpenIDConnectProvider(ctx context.Context, params *iam.CreateOpenIDConnectProviderInput, optFns ...func(*iam.Options)) (*iam.CreateOpenIDConnectProviderOutput, error) { - if m.createOidcErr { - return nil, fmt.Errorf("create Oidc error") - } - return &iam.CreateOpenIDConnectProviderOutput{OpenIDConnectProviderArn: aws.String("arn::mock")}, nil + return &iam.CreateOpenIDConnectProviderOutput{OpenIDConnectProviderArn: aws.String("arn::mock")}, m.createOidcErr } func (m *mockAwsIamAPI) DeleteOpenIDConnectProvider(ctx context.Context, params *iam.DeleteOpenIDConnectProviderInput, optFns ...func(*iam.Options)) (*iam.DeleteOpenIDConnectProviderOutput, error) { - if m.deleteOidcErr { - return nil, fmt.Errorf("delete Oidc error") - } - return &iam.DeleteOpenIDConnectProviderOutput{}, nil + return &iam.DeleteOpenIDConnectProviderOutput{}, m.deleteOidcErr } func (m *mockAwsIamAPI) CreateRole(ctx context.Context, params *iam.CreateRoleInput, optFns ...func(*iam.Options)) (*iam.CreateRoleOutput, error) { - return nil, nil + return nil, m.createRoleErr } func (m *mockAwsIamAPI) UpdateAssumeRolePolicy(ctx context.Context, params *iam.UpdateAssumeRolePolicyInput, optFns ...func(*iam.Options)) (*iam.UpdateAssumeRolePolicyOutput, error) { - return nil, nil + return nil, m.updateAssumeRolePolicyError } func (m *mockAwsIamAPI) AttachRolePolicy(ctx context.Context, params *iam.AttachRolePolicyInput, optFns ...func(*iam.Options)) (*iam.AttachRolePolicyOutput, error) { - return nil, nil + return nil, m.attachRolePolicyError } func (m *mockAwsIamAPI) DeleteRole(ctx context.Context, params *iam.DeleteRoleInput, optFns ...func(*iam.Options)) (*iam.DeleteRoleOutput, error) { - return nil, nil + return nil, m.deleteRoleErr } func (m *mockAwsIamAPI) DetachRolePolicy(ctx context.Context, params *iam.DetachRolePolicyInput, optFns ...func(*iam.Options)) (*iam.DetachRolePolicyOutput, error) { - return nil, nil + return nil, m.detachRolePolicyError } func (m *mockAwsStsAPI) GetCallerIdentity(ctx context.Context, params *sts.GetCallerIdentityInput, optFns ...func(*sts.Options)) (*sts.GetCallerIdentityOutput, error) { diff --git a/internal/manifests/serviceaccount.go b/internal/manifests/serviceaccount.go index 9fac6a3..51ef57c 100644 --- a/internal/manifests/serviceaccount.go +++ b/internal/manifests/serviceaccount.go @@ -1,6 +1,9 @@ package manifests import ( + "fmt" + + awsclient "github.com/kkb0318/irsa-manager/internal/aws" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" @@ -14,8 +17,10 @@ func NewServiceAccountBuilder() *ServiceAccountBuilder { return &ServiceAccountBuilder{} } -func (b *ServiceAccountBuilder) WithAnnotation(annotation map[string]string) *ServiceAccountBuilder { - b.annotation = annotation +func (b *ServiceAccountBuilder) WithIRSAAnnotation(role awsclient.RoleManager) *ServiceAccountBuilder { + b.annotation = map[string]string{ + "eks.amazonaws.com/role-arn": fmt.Sprintf("arn:aws:iam::%s:role/%s", role.AccountId, role.RoleName), + } return b }