Revert "Update System.Memory"

This reverts commit dcde797b21.

Directory.Build.targets

@@ -29,7 +29,7 @@
     <PackageReference Update="System.IO.Compression" Version="4.3.0" />
     <PackageReference Update="System.IO.UnmanagedMemoryStream" Version="4.3.0" />
     <PackageReference Update="System.Numerics.Vectors" Version="4.5.0" />
-    <PackageReference Update="System.Memory" Version="4.5.4" />
+    <PackageReference Update="System.Memory" Version="4.5.3" />
     <PackageReference Update="System.Runtime.CompilerServices.Unsafe" Version="4.7" />
     <PackageReference Update="System.Threading.Tasks.Parallel" Version="4.3.0" />
     <PackageReference Update="System.ValueTuple" Version="4.5.0" />