Compare commits
12 Commits
79e42e3808
...
master
| Author | SHA1 | Date | |
|---|---|---|---|
| 9939fb250f | |||
| ed85507d12 | |||
|
277c98370a | ||
|
|
12c7e047bd | ||
|
|
4a87d4d5a3 | ||
|
|
95e71a66c9 | ||
|
|
3b58aa1ef8 | ||
|
|
0cfd14f46c | ||
|
|
f1f944c1f1 | ||
|
|
267308dc0c | ||
|
|
e281d67b08 | ||
|
|
0be391f1cf |
2
.gitignore
vendored
2
.gitignore
vendored
@@ -1 +1,3 @@
|
||||
*~
|
||||
\#*
|
||||
.#*
|
||||
|
||||
@@ -1,11 +1,12 @@
|
||||
FROM docker.eenfach.de/olbohlen/kshbase:latest
|
||||
FROM quay.io/centos/centos:stream10-minimal
|
||||
LABEL author="Olaf Bohlen <olbohlen@eenfach.de>"
|
||||
|
||||
COPY receipt-processor.ksh /usr/local/bin/receipt-processor.ksh
|
||||
RUN chmod 755 /usr/local/bin/receipt-processor.ksh && \
|
||||
COPY recipe-processor.ksh /usr/local/bin/recipe-processor.ksh
|
||||
RUN chmod 755 /usr/local/bin/recipe-processor.ksh && \
|
||||
microdnf install -y shadow-utils && \
|
||||
useradd cookie && \
|
||||
dnf install -y jq
|
||||
microdnf install -y jq ksh
|
||||
|
||||
USER cookie
|
||||
|
||||
ENTRYPOINT ["/usr/local/bin/receipt-processor.ksh"]
|
||||
ENTRYPOINT ["/usr/local/bin/recipe-processor.ksh"]
|
||||
|
||||
307
README.md
Normal file
307
README.md
Normal file
@@ -0,0 +1,307 @@
|
||||
|
||||
# Table of Contents
|
||||
|
||||
1. [I need a cookie recipe database!](#org70e0ee4)
|
||||
2. [Preparing the cookierecipes CRD](#org0e8e12f)
|
||||
3. [Storing some sample recipes](#orga6bbe4b)
|
||||
4. [Creating RBAC resources](#orgde6d9d3)
|
||||
5. [Storing some sample recipes (hopefully this time!!)](#org62f1e6b)
|
||||
6. [Now can we do anything with our recipes?](#orgb60e309)
|
||||
7. [I'm an operator with my pocket calculator](#org83e0096)
|
||||
8. [What do we need?](#orgfb5d825)
|
||||
9. [Let's review the Containerfile](#orgd3ee53a)
|
||||
10. [Have a look at the Controller](#orgdd65c08)
|
||||
11. [Now let's also have a look at the deployment](#orgc7511a7)
|
||||
12. [The ServiceAccount](#org210fae4)
|
||||
13. [Building the stuff together](#org12610bc)
|
||||
14. [Deploying the Operator](#orgdd65ee3)
|
||||
15. [Test the Operator](#org27a7fba)
|
||||
16. [Test for updated recipes](#orgd947aad)
|
||||
|
||||
|
||||
<a id="org70e0ee4"></a>
|
||||
|
||||
# I need a cookie recipe database!
|
||||
|
||||
…and because it makes total sense, we are going to abuse the K8s API for it.
|
||||
|
||||
- thankfully we can extend K8s with Custom Resource Definitions (CRDs)
|
||||
- but how does it work?
|
||||
- `CustomResourceDefinitions` are themselves a `Resource`, based on a `ResourceDefinition`
|
||||
|
||||
$ oc api-resources | egrep "(NAME|CustomResourceDefinition)"
|
||||
NAME SHORTNAMES APIVERSION NAMESPACED KIND
|
||||
customresourcedefinitions crd,crds apiextensions.k8s.io/v1 false CustomResourceDefinition
|
||||
$ oc explain crds
|
||||
KIND: CustomResourceDefinition
|
||||
VERSION: apiextensions.k8s.io/v1
|
||||
|
||||
DESCRIPTION:
|
||||
CustomResourceDefinition represents a resource that should be exposed on
|
||||
the API server. Its name MUST be in the format <.spec.name>.<.spec.group>.
|
||||
[...]
|
||||
|
||||
|
||||
<a id="org0e8e12f"></a>
|
||||
|
||||
# Preparing the cookierecipes CRD
|
||||
|
||||
Let's create the CRD from <https://www.eenfach.de/gitblit/blob/~olbohlen!cookie-operator.git/master/cookie-crd.yaml>
|
||||
|
||||
$ oc new-project kitchen
|
||||
Now using project "kitchen" on server "https://api.crc.testing:6443".
|
||||
$ oc create -f cookie-crd.yaml
|
||||
Error from server (Forbidden): error when creating "cookie-crd.yaml":
|
||||
customresourcedefinitions.apiextensions.k8s.io is forbidden: User "developer"
|
||||
cannot create resource "customresourcedefinitions" in API group "apiextensions.k8s.io"
|
||||
at the cluster scope
|
||||
|
||||
$ oc login -u kubeadmin
|
||||
$ oc create -f cookie-crd.yaml
|
||||
customresourcedefinition.apiextensions.k8s.io/cookierecipes.de.eenfach.olbohlen created
|
||||
$ oc login -u developer
|
||||
|
||||
Now the cluster knows about the CRD and we could store recipes!
|
||||
|
||||
|
||||
<a id="orga6bbe4b"></a>
|
||||
|
||||
# Storing some sample recipes
|
||||
|
||||
We try to store sample cookie recipes…but:
|
||||
|
||||
$ oc create -f sample-cookie.yaml
|
||||
Error from server (Forbidden): error when creating "sample-cookie.yaml":
|
||||
cookierecipes.de.eenfach.olbohlen is forbidden: User "developer" cannot create
|
||||
resource "cookierecipes" in API group "de.eenfach.olbohlen" in the namespace "kitchen"
|
||||
Error from server (Forbidden): error when creating "sample-cookie.yaml":
|
||||
cookierecipes.de.eenfach.olbohlen is forbidden: User "developer" cannot create
|
||||
resource "cookierecipes" in API group "de.eenfach.olbohlen" in the namespace "kitchen"
|
||||
|
||||
We need to set up some RBAC resources first:
|
||||
|
||||
- a `ClusterRole` that allows viewing recipes
|
||||
- a `ClusterRole` that allows editing recipes
|
||||
- and a `ClusterRoleBinding` that allows that for authenticated users
|
||||
|
||||
|
||||
<a id="orgde6d9d3"></a>
|
||||
|
||||
# Creating RBAC resources
|
||||
|
||||
Apply the RBAC definitions from: <https://www.eenfach.de/gitblit/blob/~olbohlen!cookie-operator.git/master/cookie-rbac.yaml>
|
||||
|
||||
$ oc login -u kubeadmin
|
||||
$ oc create -f cookie-rbac.yaml
|
||||
clusterrole.rbac.authorization.k8s.io/cookierecipe-edit created
|
||||
clusterrole.rbac.authorization.k8s.io/cookierecipe-view created
|
||||
clusterrolebinding.rbac.authorization.k8s.io/cookierecipe-edit created
|
||||
|
||||
The `ClusterRoleBinding` "cookierecipe-edit" allows `system:authenticated:oauth`
|
||||
group members to edit `cookierecipes`.
|
||||
`system:authenticated:oauth` contains all users that logged in via the OAuth
|
||||
service (via an `IdentityProvider`).
|
||||
|
||||
|
||||
<a id="org62f1e6b"></a>
|
||||
|
||||
# Storing some sample recipes (hopefully this time!!)
|
||||
|
||||
Now we should be able to create the sample recipes:
|
||||
|
||||
$ oc login -u developer
|
||||
$ oc create -f sample-cookie.yaml
|
||||
cookierecipe.de.eenfach.olbohlen/vintage-chocolate-chip created
|
||||
cookierecipe.de.eenfach.olbohlen/double-dipped-shortbread created
|
||||
$ oc get cookierecipe
|
||||
NAME AGE
|
||||
double-dipped-shortbread 17s
|
||||
vintage-chocolate-chip 17s
|
||||
|
||||
There is no functionality here - we just stored the recipes in the etcd via the K8s API.
|
||||
|
||||
|
||||
<a id="orgb60e309"></a>
|
||||
|
||||
# Now can we do anything with our recipes?
|
||||
|
||||
Of course we can **oc get -o yaml** for example on them and filter:
|
||||
|
||||
$ oc get cookierecipe vintage-chocolate-chip -o yaml | yq -y .spec.ingredients[0]
|
||||
amount: 150
|
||||
name: salted butter
|
||||
remarks: softened
|
||||
unit: grams
|
||||
|
||||
This is handy, as we can extract exactly the data which we need at a time.
|
||||
|
||||
But…it's a lot of manual work…
|
||||
|
||||
|
||||
<a id="org83e0096"></a>
|
||||
|
||||
# I'm an operator with my pocket calculator
|
||||
|
||||
Operators were introduced as "Kubernetes Native Applications" and that actually
|
||||
means nothing. Operators are in the end just `Pods`.
|
||||
|
||||
These Pods run one or more containers, but one container should run a `Controller`
|
||||
that can interprete your `CustomResources`.
|
||||
|
||||
So let's write a CookieRecipe Operator. In shell-script… :)
|
||||
|
||||
Of course this Operator is not compatible with the `OperatorLifecycyleManager` (`OLM`),
|
||||
so we have to install it manually.
|
||||
|
||||
|
||||
<a id="orgfb5d825"></a>
|
||||
|
||||
# What do we need?
|
||||
|
||||
We need:
|
||||
|
||||
- a ContainerImage
|
||||
- and therefore probably a **Containerfile**
|
||||
- Controller code
|
||||
|
||||
Then we are going to build the Operator ContainerImage and push it to a Registry.
|
||||
|
||||
|
||||
<a id="orgd3ee53a"></a>
|
||||
|
||||
# Let's review the Containerfile
|
||||
|
||||
The Containerfile is here:
|
||||
|
||||
<https://www.eenfach.de/gitblit/blob/~olbohlen!cookie-operator.git/master/Containerfile>
|
||||
|
||||
the base image is a "kshbase" image, which itself is based upon ubi9 containing also a ksh93
|
||||
and an oc client.
|
||||
|
||||
|
||||
<a id="orgdd65c08"></a>
|
||||
|
||||
# Have a look at the Controller
|
||||
|
||||
The controller is written in KornShell 93 (ksh93), which is mostly bash compatible :)
|
||||
|
||||
The code is here:
|
||||
|
||||
<https://www.eenfach.de/gitblit/blob/~olbohlen!cookie-operator.git/master/recipe-processor.ksh>
|
||||
|
||||
|
||||
<a id="orgc7511a7"></a>
|
||||
|
||||
# Now let's also have a look at the deployment
|
||||
|
||||
Note: this deployment does not use an `ImageStream`, so it would work also on native k8s
|
||||
|
||||
<https://www.eenfach.de/gitblit/blob/~olbohlen!cookie-operator.git/master/cookie-operator-deployment.yaml>
|
||||
|
||||
This deployment requires a `ServiceAccount` called "cookieprocessor", this `ServiceAccount` provides
|
||||
a Token to authenticate against the API (which we use in the controller script).
|
||||
|
||||
|
||||
<a id="org210fae4"></a>
|
||||
|
||||
# The ServiceAccount
|
||||
|
||||
We need a `ServiceAccount`, but that alone will not help. The `ServiceAccount` is NOT member
|
||||
of `system:authenticated:oauth`, so it can't read `cookierecipes` based on the `ClusterRoleBinding` we created earlier.
|
||||
For that reason we also create a `RoleBinding` (namespaced!) that allows reading recipes:
|
||||
|
||||
<https://www.eenfach.de/gitblit/blob/~olbohlen!cookie-operator.git/master/cookieprocessor-sa.yaml>
|
||||
|
||||
|
||||
<a id="org12610bc"></a>
|
||||
|
||||
# Building the stuff together
|
||||
|
||||
$ oc create -f cookieprocessor-sa.yaml
|
||||
serviceaccount/cookieprocessor created
|
||||
rolebinding.rbac.authorization.k8s.io/cookierecipe-view created
|
||||
|
||||
The registry docker.eenfach.de requires login credentials, so we need to set up a secret and link it.
|
||||
**NOTE** all sample files in this repository use mirrored images on quay.io, which does not require a login.
|
||||
|
||||
First login to the registry with **podman login**, then pick the resulting auth.json:
|
||||
|
||||
$ podman login -u olbohlen docker.eenfach.de
|
||||
Password:
|
||||
Login Succeeded!
|
||||
$ oc create secret generic docker-eenfach-de \
|
||||
> --from-file=.dockerconfigjson=${XDG_RUNTIME_DIR}/containers/auth.json \
|
||||
> --type kubernetes.io/dockerconfigjson
|
||||
secret/docker-eenfach-de created
|
||||
$ oc secrets link cookieprocessor docker-eenfach-de --for pull
|
||||
|
||||
|
||||
<a id="orgdd65ee3"></a>
|
||||
|
||||
# Deploying the Operator
|
||||
|
||||
Now that we have everything in place, we will just deploy the Operator Pod:
|
||||
|
||||
$ oc create -f cookie-operator-deployment.yaml
|
||||
deployment.apps/recipe-processor created
|
||||
$ oc get pod
|
||||
NAME READY STATUS RESTARTS AGE
|
||||
recipe-processor-7f9969697b-qt9lv 1/1 Running 0 17s
|
||||
$ oc logs -f recipe-processor-7f9969697b-qt9lv
|
||||
|
||||
|
||||
New recipe found: double-dipped-shortbread
|
||||
--------------------------------------------------------------------------
|
||||
|
||||
Pre: we heat up the oven to 180 degrees Celsius
|
||||
|
||||
Fetching ingredients from recipe:
|
||||
----------------------------------
|
||||
Fetching 200grams of salted butter (softened)
|
||||
[...]
|
||||
|
||||
The Operator will process both sample recipes.
|
||||
|
||||
|
||||
<a id="org27a7fba"></a>
|
||||
|
||||
# Test the Operator
|
||||
|
||||
We should test if the Operator notices new recipes, so let's create a third
|
||||
recipe from
|
||||
<https://www.eenfach.de/gitblit/blob/~olbohlen!cookie-operator.git/master/oaty-hazelnut-cookies.yaml>
|
||||
|
||||
$ oc create -f oaty-hazelnut-cookies.yaml
|
||||
cookierecipe.de.eenfach.olbohlen/oaty-hazelnut created
|
||||
|
||||
After a few seconds, we should see in the Operator log:
|
||||
|
||||
New recipe found: oaty-hazelnut
|
||||
--------------------------------------------------------------------------
|
||||
|
||||
Pre: we heat up the oven to 180 degrees Celsius
|
||||
[...]
|
||||
|
||||
|
||||
<a id="orgd947aad"></a>
|
||||
|
||||
# Test for updated recipes
|
||||
|
||||
But what if we update a resource?
|
||||
Keep the **oc logs -f** on the Operator Pod open, and in another terminal let's patch a recipe.
|
||||
|
||||
$ oc patch cookierecipes double-dipped-shortbread --type merge \
|
||||
> -p '{"spec":{"temperature":172}}'
|
||||
cookierecipe.de.eenfach.olbohlen/double-dipped-shortbread patched
|
||||
|
||||
And again in the log you should see
|
||||
|
||||
New recipe found: double-dipped-shortbread
|
||||
--------------------------------------------------------------------------
|
||||
|
||||
Pre: we heat up the oven to 172 degrees Celsius
|
||||
|
||||
Fetching ingredients from recipe:
|
||||
----------------------------------
|
||||
[...]
|
||||
|
||||
259
README.org
Normal file
259
README.org
Normal file
@@ -0,0 +1,259 @@
|
||||
* I need a cookie recipe database!
|
||||
|
||||
...and because it makes total sense, we are going to abuse the K8s API for it.
|
||||
|
||||
- thankfully we can extend K8s with Custom Resource Definitions (CRDs)
|
||||
- but how does it work?
|
||||
- =CustomResourceDefinitions= are themselves a =Resource=, based on a =ResourceDefinition=
|
||||
|
||||
#+begin_example
|
||||
$ oc api-resources | egrep "(NAME|CustomResourceDefinition)"
|
||||
NAME SHORTNAMES APIVERSION NAMESPACED KIND
|
||||
customresourcedefinitions crd,crds apiextensions.k8s.io/v1 false CustomResourceDefinition
|
||||
$ oc explain crds
|
||||
KIND: CustomResourceDefinition
|
||||
VERSION: apiextensions.k8s.io/v1
|
||||
|
||||
DESCRIPTION:
|
||||
CustomResourceDefinition represents a resource that should be exposed on
|
||||
the API server. Its name MUST be in the format <.spec.name>.<.spec.group>.
|
||||
[...]
|
||||
#+end_example
|
||||
|
||||
* Preparing the cookierecipes CRD
|
||||
|
||||
Let's create the CRD from https://www.eenfach.de/gitblit/blob/~olbohlen!cookie-operator.git/master/cookie-crd.yaml
|
||||
|
||||
#+begin_example
|
||||
$ oc new-project kitchen
|
||||
Now using project "kitchen" on server "https://api.crc.testing:6443".
|
||||
$ oc create -f cookie-crd.yaml
|
||||
Error from server (Forbidden): error when creating "cookie-crd.yaml":
|
||||
customresourcedefinitions.apiextensions.k8s.io is forbidden: User "developer"
|
||||
cannot create resource "customresourcedefinitions" in API group "apiextensions.k8s.io"
|
||||
at the cluster scope
|
||||
|
||||
$ oc login -u kubeadmin
|
||||
$ oc create -f cookie-crd.yaml
|
||||
customresourcedefinition.apiextensions.k8s.io/cookierecipes.de.eenfach.olbohlen created
|
||||
$ oc login -u developer
|
||||
#+end_example
|
||||
|
||||
Now the cluster knows about the CRD and we could store recipes!
|
||||
|
||||
* Storing some sample recipes
|
||||
|
||||
We try to store sample cookie recipes...but:
|
||||
|
||||
#+begin_example
|
||||
$ oc create -f sample-cookie.yaml
|
||||
Error from server (Forbidden): error when creating "sample-cookie.yaml":
|
||||
cookierecipes.de.eenfach.olbohlen is forbidden: User "developer" cannot create
|
||||
resource "cookierecipes" in API group "de.eenfach.olbohlen" in the namespace "kitchen"
|
||||
Error from server (Forbidden): error when creating "sample-cookie.yaml":
|
||||
cookierecipes.de.eenfach.olbohlen is forbidden: User "developer" cannot create
|
||||
resource "cookierecipes" in API group "de.eenfach.olbohlen" in the namespace "kitchen"
|
||||
#+end_example
|
||||
|
||||
We need to set up some RBAC resources first:
|
||||
- a =ClusterRole= that allows viewing recipes
|
||||
- a =ClusterRole= that allows editing recipes
|
||||
- and a =ClusterRoleBinding= that allows that for authenticated users
|
||||
|
||||
* Creating RBAC resources
|
||||
|
||||
Apply the RBAC definitions from: https://www.eenfach.de/gitblit/blob/~olbohlen!cookie-operator.git/master/cookie-rbac.yaml
|
||||
#+begin_example
|
||||
$ oc login -u kubeadmin
|
||||
$ oc create -f cookie-rbac.yaml
|
||||
clusterrole.rbac.authorization.k8s.io/cookierecipe-edit created
|
||||
clusterrole.rbac.authorization.k8s.io/cookierecipe-view created
|
||||
clusterrolebinding.rbac.authorization.k8s.io/cookierecipe-edit created
|
||||
#+end_example
|
||||
|
||||
The =ClusterRoleBinding= "cookierecipe-edit" allows =system:authenticated:oauth=
|
||||
group members to edit =cookierecipes=.
|
||||
=system:authenticated:oauth= contains all users that logged in via the OAuth
|
||||
service (via an =IdentityProvider=).
|
||||
|
||||
* Storing some sample recipes (hopefully this time!!)
|
||||
|
||||
Now we should be able to create the sample recipes:
|
||||
|
||||
#+begin_example
|
||||
$ oc login -u developer
|
||||
$ oc create -f sample-cookie.yaml
|
||||
cookierecipe.de.eenfach.olbohlen/vintage-chocolate-chip created
|
||||
cookierecipe.de.eenfach.olbohlen/double-dipped-shortbread created
|
||||
$ oc get cookierecipe
|
||||
NAME AGE
|
||||
double-dipped-shortbread 17s
|
||||
vintage-chocolate-chip 17s
|
||||
#+end_example
|
||||
|
||||
There is no functionality here - we just stored the recipes in the etcd via the K8s API.
|
||||
|
||||
* Now can we do anything with our recipes?
|
||||
Of course we can *oc get -o yaml* for example on them and filter:
|
||||
|
||||
#+begin_example
|
||||
$ oc get cookierecipe vintage-chocolate-chip -o yaml | yq -y .spec.ingredients[0]
|
||||
amount: 150
|
||||
name: salted butter
|
||||
remarks: softened
|
||||
unit: grams
|
||||
#+end_example
|
||||
|
||||
This is handy, as we can extract exactly the data which we need at a time.
|
||||
|
||||
But...it's a lot of manual work...
|
||||
|
||||
* I'm an operator with my pocket calculator
|
||||
|
||||
Operators were introduced as "Kubernetes Native Applications" and that actually
|
||||
means nothing. Operators are in the end just =Pods=.
|
||||
|
||||
These Pods run one or more containers, but one container should run a =Controller=
|
||||
that can interprete your =CustomResources=.
|
||||
|
||||
So let's write a CookieRecipe Operator. In shell-script... :)
|
||||
|
||||
Of course this Operator is not compatible with the =OperatorLifecycyleManager= (=OLM=),
|
||||
so we have to install it manually.
|
||||
|
||||
* What do we need?
|
||||
|
||||
We need:
|
||||
- a ContainerImage
|
||||
- and therefore probably a *Containerfile*
|
||||
- Controller code
|
||||
|
||||
Then we are going to build the Operator ContainerImage and push it to a Registry.
|
||||
|
||||
* Let's review the Containerfile
|
||||
|
||||
The Containerfile is here:
|
||||
|
||||
https://www.eenfach.de/gitblit/blob/~olbohlen!cookie-operator.git/master/Containerfile
|
||||
|
||||
the base image is a "kshbase" image, which itself is based upon ubi9 containing also a ksh93
|
||||
and an oc client.
|
||||
|
||||
* Have a look at the Controller
|
||||
|
||||
The controller is written in KornShell 93 (ksh93), which is mostly bash compatible :)
|
||||
|
||||
The code is here:
|
||||
|
||||
https://www.eenfach.de/gitblit/blob/~olbohlen!cookie-operator.git/master/recipe-processor.ksh
|
||||
|
||||
* Now let's also have a look at the deployment
|
||||
|
||||
Note: this deployment does not use an =ImageStream=, so it would work also on native k8s
|
||||
|
||||
https://www.eenfach.de/gitblit/blob/~olbohlen!cookie-operator.git/master/cookie-operator-deployment.yaml
|
||||
|
||||
This deployment requires a =ServiceAccount= called "cookieprocessor", this =ServiceAccount= provides
|
||||
a Token to authenticate against the API (which we use in the controller script).
|
||||
|
||||
* The ServiceAccount
|
||||
|
||||
We need a =ServiceAccount=, but that alone will not help. The =ServiceAccount= is NOT member
|
||||
of =system:authenticated:oauth=, so it can't read =cookierecipes= based on the =ClusterRoleBinding= we created earlier.
|
||||
For that reason we also create a =RoleBinding= (namespaced!) that allows reading recipes:
|
||||
|
||||
https://www.eenfach.de/gitblit/blob/~olbohlen!cookie-operator.git/master/cookieprocessor-sa.yaml
|
||||
|
||||
* Building the stuff together
|
||||
|
||||
#+begin_example
|
||||
$ oc create -f cookieprocessor-sa.yaml
|
||||
serviceaccount/cookieprocessor created
|
||||
rolebinding.rbac.authorization.k8s.io/cookierecipe-view created
|
||||
#+end_example
|
||||
|
||||
The registry docker.eenfach.de requires login credentials, so we need to set up a secret and link it.
|
||||
*NOTE* all sample files in this repository use mirrored images on quay.io, which does not require a login.
|
||||
|
||||
First login to the registry with *podman login*, then pick the resulting auth.json:
|
||||
|
||||
#+begin_example
|
||||
$ podman login -u olbohlen docker.eenfach.de
|
||||
Password:
|
||||
Login Succeeded!
|
||||
$ oc create secret generic docker-eenfach-de \
|
||||
> --from-file=.dockerconfigjson=${XDG_RUNTIME_DIR}/containers/auth.json \
|
||||
> --type kubernetes.io/dockerconfigjson
|
||||
secret/docker-eenfach-de created
|
||||
$ oc secrets link cookieprocessor docker-eenfach-de --for pull
|
||||
#+end_example
|
||||
|
||||
* Deploying the Operator
|
||||
|
||||
Now that we have everything in place, we will just deploy the Operator Pod:
|
||||
|
||||
#+begin_example
|
||||
$ oc create -f cookie-operator-deployment.yaml
|
||||
deployment.apps/recipe-processor created
|
||||
$ oc get pod
|
||||
NAME READY STATUS RESTARTS AGE
|
||||
recipe-processor-7f9969697b-qt9lv 1/1 Running 0 17s
|
||||
$ oc logs -f recipe-processor-7f9969697b-qt9lv
|
||||
|
||||
|
||||
New recipe found: double-dipped-shortbread
|
||||
--------------------------------------------------------------------------
|
||||
|
||||
Pre: we heat up the oven to 180 degrees Celsius
|
||||
|
||||
Fetching ingredients from recipe:
|
||||
----------------------------------
|
||||
Fetching 200grams of salted butter (softened)
|
||||
[...]
|
||||
#+end_example
|
||||
|
||||
The Operator will process both sample recipes.
|
||||
|
||||
* Test the Operator
|
||||
|
||||
We should test if the Operator notices new recipes, so let's create a third
|
||||
recipe from
|
||||
https://www.eenfach.de/gitblit/blob/~olbohlen!cookie-operator.git/master/oaty-hazelnut-cookies.yaml
|
||||
|
||||
#+begin_example
|
||||
$ oc create -f oaty-hazelnut-cookies.yaml
|
||||
cookierecipe.de.eenfach.olbohlen/oaty-hazelnut created
|
||||
#+end_example
|
||||
|
||||
After a few seconds, we should see in the Operator log:
|
||||
#+begin_example
|
||||
New recipe found: oaty-hazelnut
|
||||
--------------------------------------------------------------------------
|
||||
|
||||
Pre: we heat up the oven to 180 degrees Celsius
|
||||
[...]
|
||||
#+end_example
|
||||
|
||||
* Test for updated recipes
|
||||
|
||||
But what if we update a resource?
|
||||
Keep the *oc logs -f* on the Operator Pod open, and in another terminal let's patch a recipe.
|
||||
|
||||
#+begin_example
|
||||
$ oc patch cookierecipes double-dipped-shortbread --type merge \
|
||||
> -p '{"spec":{"temperature":172}}'
|
||||
cookierecipe.de.eenfach.olbohlen/double-dipped-shortbread patched
|
||||
#+end_example
|
||||
|
||||
And again in the log you should see
|
||||
|
||||
#+begin_example
|
||||
New recipe found: double-dipped-shortbread
|
||||
--------------------------------------------------------------------------
|
||||
|
||||
Pre: we heat up the oven to 172 degrees Celsius
|
||||
|
||||
Fetching ingredients from recipe:
|
||||
----------------------------------
|
||||
[...]
|
||||
#+end_example
|
||||
@@ -1,13 +1,15 @@
|
||||
apiVersion: apiextensions.k8s.io/v1
|
||||
kind: CustomResourceDefinition
|
||||
metadata:
|
||||
name: cookiereceipts.de.eenfach.olbohlen
|
||||
name: cookierecipes.de.eenfach.olbohlen
|
||||
spec:
|
||||
group: de.eenfach.olbohlen
|
||||
names:
|
||||
kind: CookieReceipt
|
||||
plural: cookiereceipts
|
||||
singular: cookiereceipt
|
||||
kind: CookieRecipe
|
||||
plural: cookierecipes
|
||||
singular: cookierecipe
|
||||
shortNames:
|
||||
- cr
|
||||
scope: Namespaced
|
||||
versions:
|
||||
- name: v1
|
||||
@@ -15,7 +17,7 @@ spec:
|
||||
storage: true
|
||||
schema:
|
||||
openAPIV3Schema:
|
||||
description: "The CookieReceipt CRD is a k8s demo for enhancing functionality, it will not (unfortunately) provide you real cookies in the end..."
|
||||
description: "The CookieRecipe CRD is a k8s demo for enhancing functionality, it will not (unfortunately) provide you real cookies in the end..."
|
||||
type: object
|
||||
properties:
|
||||
apiVersion:
|
||||
@@ -40,7 +42,7 @@ spec:
|
||||
type: string
|
||||
description: where does this come from? (grandma, mum, the internet...)
|
||||
ingredients:
|
||||
description: this list provides required ingredients for this receipt
|
||||
description: this list provides required ingredients for this recipe
|
||||
type: array
|
||||
x-kubernetes-list-type: atomic
|
||||
items:
|
||||
|
||||
@@ -3,23 +3,23 @@ kind: Deployment
|
||||
metadata:
|
||||
creationTimestamp: null
|
||||
labels:
|
||||
app: receipt-processor
|
||||
name: receipt-processor
|
||||
app: recipe-processor
|
||||
name: recipe-processor
|
||||
spec:
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app: receipt-processor
|
||||
app: recipe-processor
|
||||
strategy: {}
|
||||
template:
|
||||
metadata:
|
||||
creationTimestamp: null
|
||||
labels:
|
||||
app: receipt-processor
|
||||
app: recipe-processor
|
||||
spec:
|
||||
serviceAccountName: cookieprocessor
|
||||
containers:
|
||||
- image: docker.eenfach.de/olbohlen/cookie-operator:latest
|
||||
- image: quay.io/kareiva/cookie-operator:latest
|
||||
name: cookie-operator
|
||||
resources: {}
|
||||
status: {}
|
||||
|
||||
@@ -2,12 +2,12 @@
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: ClusterRole
|
||||
metadata:
|
||||
name: cookiereceipt-edit
|
||||
name: cookierecipe-edit
|
||||
rules:
|
||||
- apiGroups:
|
||||
- de.eenfach.olbohlen
|
||||
resources:
|
||||
- cookiereceipts
|
||||
- cookierecipes
|
||||
verbs:
|
||||
- create
|
||||
- get
|
||||
@@ -19,12 +19,12 @@ rules:
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: ClusterRole
|
||||
metadata:
|
||||
name: cookiereceipt-view
|
||||
name: cookierecipe-view
|
||||
rules:
|
||||
- apiGroups:
|
||||
- de.eenfach.olbohlen
|
||||
resources:
|
||||
- cookiereceipts
|
||||
- cookierecipes
|
||||
verbs:
|
||||
- get
|
||||
- list
|
||||
@@ -32,11 +32,11 @@ rules:
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: ClusterRoleBinding
|
||||
metadata:
|
||||
name: cookiereceipt-edit
|
||||
name: cookierecipe-edit
|
||||
roleRef:
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
kind: ClusterRole
|
||||
name: cookiereceipt-edit
|
||||
name: cookierecipe-edit
|
||||
subjects:
|
||||
- apiGroup: rbac.authorization.k8s.io
|
||||
kind: Group
|
||||
|
||||
@@ -8,11 +8,11 @@ metadata:
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: RoleBinding
|
||||
metadata:
|
||||
name: cookiereceipt-view
|
||||
name: cookierecipe-view
|
||||
roleRef:
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
kind: ClusterRole
|
||||
name: cookiereceipt-view
|
||||
name: cookierecipe-view
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: cookieprocessor
|
||||
|
||||
62
oaty-hazelnut-cookies.yaml
Normal file
62
oaty-hazelnut-cookies.yaml
Normal file
@@ -0,0 +1,62 @@
|
||||
apiVersion: de.eenfach.olbohlen/v1
|
||||
kind: CookieRecipe
|
||||
metadata:
|
||||
name: oaty-hazelnut
|
||||
spec:
|
||||
description: >-
|
||||
Soft and slightly chewy, these oaty cookies contain apple and maple syrup
|
||||
instead of sugar. They're packed with hazelnuts which are a good source of
|
||||
vitamins and minerals
|
||||
source: https://www.bbcgoodfood.com/recipes/oaty-hazelnut-cookies
|
||||
temperature: 180
|
||||
sanescale: true
|
||||
preheat: true
|
||||
duration: 18
|
||||
ingredients:
|
||||
- name: butter
|
||||
amount: 50
|
||||
unit: grams
|
||||
remarks: plus a little for greasing
|
||||
- name: maple syrup
|
||||
amount: 2
|
||||
unit: table spoons
|
||||
- name: dessert apple
|
||||
amount: 1
|
||||
unit: piece
|
||||
remarks: unpeeled and coarsely grated, you need 85g
|
||||
- name: cinnamon
|
||||
amount: 1
|
||||
unit: tea spoon
|
||||
- name: raisins
|
||||
amount: 50
|
||||
unit: grams
|
||||
- name: porridge oats
|
||||
amount: 50
|
||||
unit: grams
|
||||
- name: spelt flour
|
||||
amount: 50
|
||||
unit: grams
|
||||
- name: unblanched hazelnuts
|
||||
amount: 40
|
||||
unit: grams
|
||||
remarks: cut into chunky slices
|
||||
- name: egg
|
||||
amount: 1
|
||||
unit: pieces
|
||||
steps:
|
||||
- order: 1
|
||||
instruction: >-
|
||||
Heat oven to 180C/160C fan/gas 4 and lightly grease a non-stick baking tray
|
||||
(or line a normal baking tray with baking parchment). Tip the butter and syrup
|
||||
into a small non-stick pan and melt together, then add the apple and cook,
|
||||
stirring, over a medium heat until it softens, about 6-7 mins. Stir in the
|
||||
cinnamon and raisins.
|
||||
- order: 2
|
||||
instruction: >-
|
||||
Mix the oats, spelt flour, and hazelnuts in a bowl, pour in the apple mixture,
|
||||
then add the egg and beat everything together really well.
|
||||
- order: 3
|
||||
instruction: >-
|
||||
Spoon onto the baking tray, well spaced apart to make 9 mounds, then gently
|
||||
press into discs. Bake for 18-20 mins until golden, then cool on a wire rack.
|
||||
Will keep for 3 days in an airtight container or 6 weeks in the freezer.
|
||||
@@ -1,124 +0,0 @@
|
||||
#!/usr/bin/ksh
|
||||
# by Olaf Bohlen <olbohlen@eenfach.de>
|
||||
# licensed under BSD3 license
|
||||
|
||||
APIURL="https://${KUBERNETES_SERVICE_HOST}:${KUBERNETES_SERVICE_PORT_HTTPS}"
|
||||
NAMESPACE=$(</run/secrets/kubernetes.io/serviceaccount/namespace)
|
||||
TOKEN=$(</run/secrets/kubernetes.io/serviceaccount/token)
|
||||
CACRT="/run/secrets/kubernetes.io/serviceaccount/ca.crt"
|
||||
|
||||
# some associate arrays we need
|
||||
typeset -A receiptname
|
||||
typeset -A receiptversion
|
||||
|
||||
CURLOPTS="-s --cacert ${CACRT} -H 'Accept: application/json' -H 'User-Agent: ksh93 receipt processor 0.1' -H 'Authorization: Bearer ${TOKEN}'"
|
||||
|
||||
|
||||
while sleep 5; do
|
||||
# MAIN loop
|
||||
|
||||
# we will later store here modified receipt uids, so we clear it for every run
|
||||
updatedreceipts=""
|
||||
|
||||
# get a CookieReceiptList item from the API and store the json in a variable
|
||||
respjson=$(eval curl -XGET ${CURLOPTS} "${APIURL}/apis/de.eenfach.olbohlen/v1/namespaces/${NAMESPACE}/cookiereceipts?limit=500")
|
||||
|
||||
# check if we got the expected resource:
|
||||
kind=$(echo "${respjson}" | jq .kind)
|
||||
if [ x${kind} != 'x"CookieReceiptList"' ]; then
|
||||
printf "Error: unexpected Result from API\n"
|
||||
break ## jump out of this loop iteration
|
||||
fi
|
||||
|
||||
# how many receipts are there?
|
||||
numreceipts=$(echo "${respjson}" | jq .items\[\].metadata.name | wc -l)
|
||||
|
||||
# populate the in memory arrays with json metadata, we need
|
||||
# - the name
|
||||
# - the uid
|
||||
# - the resourceVersion
|
||||
# for every receipt to figure out if we need to process them
|
||||
i=0
|
||||
while [ ${i} -lt ${numreceipts} ]; do
|
||||
receiptuid=$(echo "${respjson}" | jq -r .items[${i}].metadata.uid)
|
||||
receiptname[${receiptuid}]=$(echo "${respjson}" | jq -r .items[${i}].metadata.name)
|
||||
|
||||
# check if we already have processed this version of the receipt, if not
|
||||
# store the uid of that receipt in the updatedreceipts var
|
||||
newversion=$(echo "${respjson}" | jq -r .items[${i}].metadata.resourceVersion)
|
||||
if [ "x${newversion}" != "x${receiptversion[${receiptuid}]}" ]; then
|
||||
# we have an update!
|
||||
updatedreceipts="${updatedreceipts} ${receiptuid}"
|
||||
receiptversion[${receiptuid}]="${newversion}"
|
||||
fi
|
||||
i=$(( ${i} + 1 ))
|
||||
done
|
||||
|
||||
# now that we have a list of updated receipts, we are going to process them
|
||||
for r in ${updatedreceipts}; do
|
||||
printf "\n\nNew receipt found: %s\n" "${receiptname[${r}]}"
|
||||
printf "--------------------------------------------------------------------------\n\n"
|
||||
|
||||
# get the name for the UID and fetch only that object
|
||||
receipt=$(eval curl -XGET ${CURLOPTS} '${APIURL}/apis/de.eenfach.olbohlen/v1/namespaces/${NAMESPACE}/cookiereceipts/${receiptname[${r}]}' )
|
||||
|
||||
# receipt contains now the json for one receipt, now we parse that
|
||||
# we set scale to Fahrenheit if sanescale is false, else scale will use Celsius
|
||||
scale_b=$( echo "${receipt}" | jq -r .spec.sanescale )
|
||||
if [ "x${sanescale}" == "xfalse" ]; then
|
||||
scale=Fahrenheit
|
||||
fi
|
||||
|
||||
temperature=$( echo "${receipt}" | jq -r .spec.temperature )
|
||||
|
||||
preheat_b=$( echo "${receipt}" | jq -r .spec.preheat )
|
||||
if [ "x${preheat_b}" == "xtrue" ]; then
|
||||
printf "Pre: we heat up the oven to %s degrees %s\n\n" "${temperature}" "${scale:-Celsius}"
|
||||
fi
|
||||
|
||||
# how many ingredients do we have?
|
||||
num_in=$(echo "${receipt}" | jq .spec.ingredients\[\].name | wc -l)
|
||||
|
||||
# list up that we fetch needed ingredients
|
||||
printf "Fetching ingredients from receipt:\n"
|
||||
i=0
|
||||
while [ ${i} -lt ${num_in} ]; do
|
||||
in_name=$( echo "${receipt}" | jq -r .spec.ingredients[${i}].name )
|
||||
in_amount=$( echo "${receipt}" | jq -r .spec.ingredients[${i}].amount )
|
||||
in_unit=$( echo "${receipt}" | jq -r .spec.ingredients[${i}].unit )
|
||||
in_remarks=$( echo "${receipt}" | jq -r .spec.ingredients[${i}].remarks )
|
||||
|
||||
printf "Fetching %s%s of %s" "${in_amount}" "${in_unit}" "${in_name}"
|
||||
if [ "x${in_remarks}" != "xnull" ]; then
|
||||
printf " (%s)" "${in_remarks}"
|
||||
fi
|
||||
printf "\n"
|
||||
sleep 1
|
||||
i=$(( ${i} + 1 ))
|
||||
done
|
||||
|
||||
|
||||
# now we need to (unfortunately just!) simulate the processing
|
||||
# the order of the steps is important, but lists are not ordered, so we have
|
||||
# an "order" attribute in each list item, and we select on that.
|
||||
# first again, we need the amount of instructions...
|
||||
|
||||
num_steps=$(echo "${receipt}" | jq .spec.steps\[\].order | wc -l)
|
||||
|
||||
# now let's iterate over that
|
||||
printf "\n\nProcessing the instructions:\n"
|
||||
i=1
|
||||
while [ ${i} -lt ${num_steps} ]; do
|
||||
instruction=$( echo "${receipt}" | jq '.spec.steps[] | select(.order == '${i}') | { instruction } | join (" ")' )
|
||||
|
||||
printf "Step %i/%i: %s..." ${i} ${num_steps} "${instruction}"
|
||||
sleep $(( ${RANDOM} % 6 + 1 ))
|
||||
printf "done\n"
|
||||
sleep 1
|
||||
i=$(( ${i} + 1 ))
|
||||
done
|
||||
printf "\nDone with this receipt."
|
||||
printf "\n=======================\n\n"
|
||||
done
|
||||
done
|
||||
|
||||
126
recipe-processor.ksh
Normal file
126
recipe-processor.ksh
Normal file
@@ -0,0 +1,126 @@
|
||||
#!/usr/bin/ksh
|
||||
# by Olaf Bohlen <olbohlen@eenfach.de>
|
||||
# licensed under BSD3 license
|
||||
|
||||
APIURL="https://${KUBERNETES_SERVICE_HOST}:${KUBERNETES_SERVICE_PORT_HTTPS}"
|
||||
NAMESPACE=$(</run/secrets/kubernetes.io/serviceaccount/namespace)
|
||||
TOKEN=$(</run/secrets/kubernetes.io/serviceaccount/token)
|
||||
CACRT="/run/secrets/kubernetes.io/serviceaccount/ca.crt"
|
||||
|
||||
# some associate arrays we need
|
||||
typeset -A recipename
|
||||
typeset -A recipeversion
|
||||
|
||||
CURLOPTS="-s --cacert ${CACRT} -H 'Accept: application/json' -H 'User-Agent: ksh93 recipe processor 0.1' -H 'Authorization: Bearer ${TOKEN}'"
|
||||
|
||||
|
||||
while sleep 5; do
|
||||
# MAIN loop
|
||||
|
||||
# we will later store here modified recipe uids, so we clear it for every run
|
||||
updatedrecipes=""
|
||||
|
||||
# get a CookieRecipeList item from the API and store the json in a variable
|
||||
respjson=$(eval curl -XGET ${CURLOPTS} "${APIURL}/apis/de.eenfach.olbohlen/v1/namespaces/${NAMESPACE}/cookierecipes?limit=500")
|
||||
|
||||
# check if we got the expected resource:
|
||||
kind=$(echo "${respjson}" | jq .kind)
|
||||
if [ x${kind} != 'x"CookieRecipeList"' ]; then
|
||||
printf "Error: unexpected Result from API\n"
|
||||
break ## jump out of this loop iteration
|
||||
fi
|
||||
|
||||
# how many recipes are there?
|
||||
numrecipes=$(echo "${respjson}" | jq .items\[\].metadata.name | wc -l)
|
||||
|
||||
# populate the in memory arrays with json metadata, we need
|
||||
# - the name
|
||||
# - the uid
|
||||
# - the resourceVersion
|
||||
# for every recipe to figure out if we need to process them
|
||||
i=0
|
||||
while [ ${i} -lt ${numrecipes} ]; do
|
||||
recipeuid=$(echo "${respjson}" | jq -r .items[${i}].metadata.uid)
|
||||
recipename[${recipeuid}]=$(echo "${respjson}" | jq -r .items[${i}].metadata.name)
|
||||
|
||||
# check if we already have processed this version of the recipe, if not
|
||||
# store the uid of that recipe in the updatedrecipes var
|
||||
newversion=$(echo "${respjson}" | jq -r .items[${i}].metadata.resourceVersion)
|
||||
if [ "x${newversion}" != "x${recipeversion[${recipeuid}]}" ]; then
|
||||
# we have an update!
|
||||
updatedrecipes="${updatedrecipes} ${recipeuid}"
|
||||
recipeversion[${recipeuid}]="${newversion}"
|
||||
fi
|
||||
i=$(( ${i} + 1 ))
|
||||
done
|
||||
|
||||
# now that we have a list of updated recipes, we are going to process them
|
||||
for r in ${updatedrecipes}; do
|
||||
printf "\n\nNew recipe found: %s\n" "${recipename[${r}]}"
|
||||
printf "--------------------------------------------------------------------------\n\n"
|
||||
|
||||
# get the name for the UID and fetch only that object
|
||||
recipe=$(eval curl -XGET ${CURLOPTS} '${APIURL}/apis/de.eenfach.olbohlen/v1/namespaces/${NAMESPACE}/cookierecipes/${recipename[${r}]}' )
|
||||
|
||||
# recipe contains now the json for one recipe, now we parse that
|
||||
# we set scale to Fahrenheit if sanescale is false, else scale will use Celsius
|
||||
scale_b=$( echo "${recipe}" | jq -r .spec.sanescale )
|
||||
if [ "x${sanescale}" == "xfalse" ]; then
|
||||
scale=Fahrenheit
|
||||
fi
|
||||
|
||||
temperature=$( echo "${recipe}" | jq -r .spec.temperature )
|
||||
|
||||
preheat_b=$( echo "${recipe}" | jq -r .spec.preheat )
|
||||
if [ "x${preheat_b}" == "xtrue" ]; then
|
||||
printf "Pre: we heat up the oven to %s degrees %s\n\n" "${temperature}" "${scale:-Celsius}"
|
||||
fi
|
||||
|
||||
# how many ingredients do we have?
|
||||
num_in=$(echo "${recipe}" | jq .spec.ingredients\[\].name | wc -l)
|
||||
|
||||
# list up that we fetch needed ingredients
|
||||
printf "Fetching ingredients from recipe:\n"
|
||||
printf "----------------------------------\n"
|
||||
i=0
|
||||
while [ ${i} -lt ${num_in} ]; do
|
||||
in_name=$( echo "${recipe}" | jq -r .spec.ingredients[${i}].name )
|
||||
in_amount=$( echo "${recipe}" | jq -r .spec.ingredients[${i}].amount )
|
||||
in_unit=$( echo "${recipe}" | jq -r .spec.ingredients[${i}].unit )
|
||||
in_remarks=$( echo "${recipe}" | jq -r .spec.ingredients[${i}].remarks )
|
||||
|
||||
printf "Fetching %s%s of %s" "${in_amount}" "${in_unit}" "${in_name}"
|
||||
if [ "x${in_remarks}" != "xnull" ]; then
|
||||
printf " (%s)" "${in_remarks}"
|
||||
fi
|
||||
printf "\n\n"
|
||||
sleep 1
|
||||
i=$(( ${i} + 1 ))
|
||||
done
|
||||
|
||||
|
||||
# now we need to (unfortunately just!) simulate the processing
|
||||
# the order of the steps is important, but lists are not ordered, so we have
|
||||
# an "order" attribute in each list item, and we select on that.
|
||||
# first again, we need the amount of instructions...
|
||||
|
||||
num_steps=$(echo "${recipe}" | jq .spec.steps\[\].order | wc -l)
|
||||
|
||||
# now let's iterate over that
|
||||
printf "\n\nProcessing the instructions:\n"
|
||||
printf "----------------------------\n"
|
||||
i=1
|
||||
while [ ${i} -le ${num_steps} ]; do
|
||||
instruction=$( echo "${recipe}" | jq '.spec.steps[] | select(.order == '${i}') | { instruction } | join (" ")' )
|
||||
|
||||
printf "Step %i/%i: %s..." ${i} ${num_steps} "${instruction}"
|
||||
sleep $(( ${RANDOM} % 6 + 1 ))
|
||||
printf "done\n\n"
|
||||
sleep 1
|
||||
i=$(( ${i} + 1 ))
|
||||
done
|
||||
printf "\nDone with this recipe."
|
||||
printf "\n=======================\n\n"
|
||||
done
|
||||
done
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
apiVersion: de.eenfach.olbohlen/v1
|
||||
kind: CookieReceipt
|
||||
kind: CookieRecipe
|
||||
metadata:
|
||||
name: vintage-chocolate-chip
|
||||
spec:
|
||||
@@ -62,7 +62,7 @@ spec:
|
||||
instruction: Leave on the tray for a couple of mins to set and then lift onto a cooling rack.
|
||||
---
|
||||
apiVersion: de.eenfach.olbohlen/v1
|
||||
kind: CookieReceipt
|
||||
kind: CookieRecipe
|
||||
metadata:
|
||||
name: double-dipped-shortbread
|
||||
spec:
|
||||
|
||||
Reference in New Issue
Block a user