diff --git a/Dockerfile b/Dockerfile
index 3ddc3c8..421bca9 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -8,4 +8,5 @@ RUN cargo build --release
 FROM debian:stable-slim
 WORKDIR /app
 COPY --from=builder /app/target/release/handler .
+COPY --from=builder /app/target/release/skubelb .
 CMD ["./handler"]
\ No newline at end of file
diff --git a/src/rewriter.rs b/src/rewriter.rs
index 9317c37..4c50c2f 100644
--- a/src/rewriter.rs
+++ b/src/rewriter.rs
@@ -98,7 +98,7 @@ impl Rewriter {
         Ok(())
     }
 
-    pub fn cleanup(&mut self) {
+    pub fn cleanup(&mut self) -> bool {
         let now = Instant::now();
         let mut to_remove = vec![];
         for (name, when) in self.replacement_cleanup.iter() {
@@ -106,10 +106,11 @@ impl Rewriter {
                 to_remove.push(name.clone());
             }
         }
-
+        let will_cleanup = to_remove.len() > 0;
         for name in to_remove {
             self.remove_replacement(&name);
         }
+        will_cleanup
     }
 }
 
diff --git a/src/server.rs b/src/server.rs
index d1acf23..0715096 100644
--- a/src/server.rs
+++ b/src/server.rs
@@ -36,9 +36,12 @@ impl Server {
         }
     }
 
-    pub fn cleanup(&mut self) -> Result<()> {
-        self.rewriter.cleanup();
-        Ok(())
+    pub fn cleanup(&mut self) -> Result<bool> {
+        let cleaned_up = self.rewriter.cleanup();
+        if cleaned_up {
+            self.generate_config()?;
+        }
+        Ok(cleaned_up)
     }
 
     pub fn register(&mut self, _request: &Request, ip: &str) -> Result<()> {